Skip to main content

strided_basic/
map_view.rs

1//! Map operations on dynamic-rank strided views.
2//!
3//! These are the canonical view-based map functions, equivalent to Julia's `Base.map!`.
4//! Mutable destinations must have an injective layout. Validation is
5//! conservative and bounded: layouts that are injective but cannot be proven
6//! by the bounded checker are rejected with
7//! [`StridedError::NonInjectiveOutputLayout`].
8
9use crate::kernel::{
10    ensure_same_shape, for_each_inner_block_preordered, pointwise_plan,
11    sequential_contiguous_layout, total_len,
12};
13use crate::simd;
14use crate::view::{StridedView, StridedViewMut};
15use crate::{Result, StridedError};
16use core::mem::MaybeUninit;
17use std::ops::Mul;
18use strided_view::{ElementOp, Identity};
19
20use crate::cost::{for_len, KernelClass};
21use crate::fuse::compute_costs;
22use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
23use crate::Exec;
24
25type AxisVec<T> = crate::inline::InlineVec<T>;
26
27/// Largest fused inner:row aspect at which the tiled transposed-scalar kernel
28/// is selected.
29///
30/// The cache-tiled kernel wins or ties the generic blocked plan across the
31/// measured shape/dtype table (u4-tiled #210), except when the fused inner run
32/// is more than 8x the row extent: then its strided destination writes lose to
33/// the blocked plan on f64 (`4096x256` ~1.4x, `8192x256` ~1.3x at 1T), so the
34/// detector declines and the generic path runs.
35const TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT: usize = 8;
36
37/// Internal validation marker used by kernel-family execution adapters.
38///
39/// This marker is not bound to a destination. It does not authorize unchecked
40/// replay with a different geometry; callers of the unsafe execution entries
41/// must establish that association themselves.
42#[derive(Clone, Copy, Debug, Default)]
43pub struct ValidatedDestinationLayout(());
44
45/// Validate destination injectivity and return the replay marker.
46///
47/// Runs the exact (allocation-free pairwise, then spilling) injectivity check,
48/// so every injective layout the other kernel entry points accept is accepted
49/// here. Kernel families that need the marker for more than one step (for
50/// example validating shapes and then taking a specialization) should call this
51/// once and pass the marker on instead of re-running the check.
52///
53/// # Errors
54///
55/// Returns [`StridedError::NonInjectiveOutputLayout`] when the layout cannot
56/// be proven representable and injective: two distinct logical indices of
57/// `dims` name the same offset under `strides`, or the element count or the
58/// address span is too large to represent. A rejected layout need not be
59/// colliding.
60#[inline]
61pub fn validate_destination_layout(
62    dims: &[usize],
63    strides: &[isize],
64) -> Result<ValidatedDestinationLayout> {
65    if crate::layout_check::is_injective_layout(dims, strides) {
66        Ok(ValidatedDestinationLayout(()))
67    } else {
68        Err(StridedError::NonInjectiveOutputLayout)
69    }
70}
71
72#[inline]
73/// Validate injectivity without allocating a set of offsets.
74///
75/// # Examples
76///
77/// ```
78/// use strided_basic::execution::validate_destination_layout_without_alloc;
79/// let marker = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
80/// assert!(validate_destination_layout_without_alloc(&[2], &[0]).is_err());
81/// let _ = marker;
82/// ```
83///
84/// # Errors
85/// Returns `NonInjectiveOutputLayout` for invalid or unsupported output geometry.
86pub fn validate_destination_layout_without_alloc(
87    dims: &[usize],
88    strides: &[isize],
89) -> Result<ValidatedDestinationLayout> {
90    if crate::layout_check::is_injective_layout_without_alloc(dims, strides) {
91        Ok(ValidatedDestinationLayout(()))
92    } else {
93        Err(StridedError::NonInjectiveOutputLayout)
94    }
95}
96
97fn reachable_byte_range(
98    ptr: usize,
99    elem_size: usize,
100    dims: &[usize],
101    strides: &[isize],
102) -> Result<Option<(usize, usize)>> {
103    if elem_size == 0 || dims.contains(&0) {
104        return Ok(None);
105    }
106
107    let mut min_offset = 0isize;
108    let mut max_offset = 0isize;
109    for (&dim, &stride) in dims.iter().zip(strides) {
110        if dim <= 1 {
111            continue;
112        }
113        let extent = isize::try_from(dim - 1).map_err(|_| StridedError::OffsetOverflow)?;
114        let span = stride
115            .checked_mul(extent)
116            .ok_or(StridedError::OffsetOverflow)?;
117        if span < 0 {
118            min_offset = min_offset
119                .checked_add(span)
120                .ok_or(StridedError::OffsetOverflow)?;
121        } else {
122            max_offset = max_offset
123                .checked_add(span)
124                .ok_or(StridedError::OffsetOverflow)?;
125        }
126    }
127
128    let ptr = ptr as i128;
129    let elem_size = elem_size as i128;
130    let start = ptr
131        .checked_add(
132            (min_offset as i128)
133                .checked_mul(elem_size)
134                .ok_or(StridedError::OffsetOverflow)?,
135        )
136        .ok_or(StridedError::OffsetOverflow)?;
137    let end = ptr
138        .checked_add(
139            (max_offset as i128)
140                .checked_mul(elem_size)
141                .and_then(|offset| offset.checked_add(elem_size))
142                .ok_or(StridedError::OffsetOverflow)?,
143        )
144        .ok_or(StridedError::OffsetOverflow)?;
145    if start < 0 || end < 0 || start > usize::MAX as i128 || end > usize::MAX as i128 {
146        return Err(StridedError::OffsetOverflow);
147    }
148    Ok(Some((start as usize, end as usize)))
149}
150
151fn validate_typed_no_overlap<D, A, Op: ElementOp<A>>(
152    dest: &StridedViewMut<MaybeUninit<D>>,
153    input: &StridedView<A, Op>,
154    input_index: usize,
155) -> Result<()> {
156    let dest_range = reachable_byte_range(
157        dest.ptr() as usize,
158        core::mem::size_of::<D>(),
159        dest.dims(),
160        dest.strides(),
161    )?;
162    let input_range = reachable_byte_range(
163        input.ptr() as usize,
164        core::mem::size_of::<A>(),
165        input.dims(),
166        input.strides(),
167    )?;
168    if let (Some((dest_start, dest_end)), Some((input_start, input_end))) =
169        (dest_range, input_range)
170    {
171        if dest_start < input_end && input_start < dest_end {
172            return Err(StridedError::OverlappingInputOutput { input: input_index });
173        }
174    }
175    Ok(())
176}
177
178// ============================================================================
179// Stride-specialized inner loop helpers
180//
181// When all inner strides are 1 (contiguous in the innermost dimension),
182// we use slice-based iteration so LLVM can auto-vectorize effectively.
183// This is the Rust equivalent of Julia's @simd on the innermost loop.
184// ============================================================================
185
186// INVARIANT: validated layouts keep every dereferenced cursor in bounds.
187// Pointwise leaves use wrapping advances: the unused final cursor may lie
188// outside the allocation for negative or gapped strides.
189/// Unary inner loop: `dest[i] = f(Op::apply(src[i]))` for `len` elements.
190///
191/// A vectorized kernel a map operation offers its flat runs to.
192///
193/// [`NoKernel`] is what the entry points without a kernel pass, so the check it introduces is
194/// monomorphized away; [`SliceKernel`] adapts the closure a caller supplied.
195trait MapKernel<A, D> {
196    /// Writes every element of `dst` from `src`, or writes nothing and returns `false`.
197    fn apply(&self, src: &[A], dst: &mut [D]) -> bool;
198}
199
200struct NoKernel;
201
202impl<A, D> MapKernel<A, D> for NoKernel {
203    #[inline(always)]
204    fn apply(&self, _src: &[A], _dst: &mut [D]) -> bool {
205        false
206    }
207}
208
209struct SliceKernel<K>(K);
210
211impl<A, D, K: Fn(&[A], &mut [D]) -> bool> MapKernel<A, D> for SliceKernel<K> {
212    #[inline(always)]
213    fn apply(&self, src: &[A], dst: &mut [D]) -> bool {
214        (self.0)(src, dst)
215    }
216}
217
218unsafe fn inner_loop_map1<D: Copy, A: Copy, Op: ElementOp<A>, K: MapKernel<A, D>>(
219    dp: *mut D,
220    ds: isize,
221    sp: *const A,
222    ss: isize,
223    len: usize,
224    f: &impl Fn(A) -> D,
225    kernel: &K,
226) {
227    if ds == 1 && ss == 1 {
228        let src = std::slice::from_raw_parts(sp, len);
229        let dst = std::slice::from_raw_parts_mut(dp, len);
230        if kernel.apply(src, dst) {
231            return;
232        }
233        simd::dispatch_if_large(len, || {
234            for (d, s) in dst.iter_mut().zip(src.iter()) {
235                *d = f(Op::apply(*s));
236            }
237        });
238    } else {
239        let mut dp = dp;
240        let mut sp = sp;
241        simd::dispatch_if_large(len, || {
242            for _ in 0..len {
243                *dp = f(Op::apply(*sp));
244                dp = dp.wrapping_offset(ds);
245                sp = sp.wrapping_offset(ss);
246            }
247        });
248    }
249}
250
251/// Binary inner loop: `dest[i] = f(OpA::apply(a[i]), OpB::apply(b[i]))`.
252// Two operand pointers with their strides, the destination, the length and the
253// element op: the call's inputs, not a block's.
254#[allow(clippy::too_many_arguments)]
255#[inline(always)]
256unsafe fn inner_loop_map2<D: Copy, A: Copy, B: Copy, OpA: ElementOp<A>, OpB: ElementOp<B>>(
257    dp: *mut D,
258    ds: isize,
259    ap: *const A,
260    a_s: isize,
261    bp: *const B,
262    b_s: isize,
263    len: usize,
264    f: &impl Fn(A, B) -> D,
265) {
266    if ds == 1 && a_s == 1 && b_s == 1 {
267        let src_a = std::slice::from_raw_parts(ap, len);
268        let src_b = std::slice::from_raw_parts(bp, len);
269        let dst = std::slice::from_raw_parts_mut(dp, len);
270        simd::dispatch_if_large(len, || {
271            for i in 0..len {
272                dst[i] = f(OpA::apply(src_a[i]), OpB::apply(src_b[i]));
273            }
274        });
275    } else if ds == 1 && a_s == 1 && b_s == 0 {
276        let src_a = std::slice::from_raw_parts(ap, len);
277        let b = OpB::apply(*bp);
278        let dst = std::slice::from_raw_parts_mut(dp, len);
279        simd::dispatch_if_large(len, || {
280            for i in 0..len {
281                dst[i] = f(OpA::apply(src_a[i]), b);
282            }
283        });
284    } else if ds == 1 && a_s == 0 && b_s == 1 {
285        let a = OpA::apply(*ap);
286        let src_b = std::slice::from_raw_parts(bp, len);
287        let dst = std::slice::from_raw_parts_mut(dp, len);
288        simd::dispatch_if_large(len, || {
289            for i in 0..len {
290                dst[i] = f(a, OpB::apply(src_b[i]));
291            }
292        });
293    } else if ds == 1 && a_s == 0 && b_s == 0 {
294        let a = OpA::apply(*ap);
295        let b = OpB::apply(*bp);
296        let dst = std::slice::from_raw_parts_mut(dp, len);
297        simd::dispatch_if_large(len, || {
298            for d in dst.iter_mut() {
299                *d = f(a, b);
300            }
301        });
302    } else if ds == 1 && b_s == 0 {
303        let b = OpB::apply(*bp);
304        let dst = std::slice::from_raw_parts_mut(dp, len);
305        let mut ap = ap;
306        simd::dispatch_if_large(len, || {
307            for d in dst.iter_mut() {
308                *d = f(OpA::apply(*ap), b);
309                ap = ap.wrapping_offset(a_s);
310            }
311        });
312    } else if ds == 1 && a_s == 0 {
313        let a = OpA::apply(*ap);
314        let dst = std::slice::from_raw_parts_mut(dp, len);
315        let mut bp = bp;
316        simd::dispatch_if_large(len, || {
317            for d in dst.iter_mut() {
318                *d = f(a, OpB::apply(*bp));
319                bp = bp.wrapping_offset(b_s);
320            }
321        });
322    } else {
323        let mut dp = dp;
324        let mut ap = ap;
325        let mut bp = bp;
326        simd::dispatch_if_large(len, || {
327            for _ in 0..len {
328                *dp = f(OpA::apply(*ap), OpB::apply(*bp));
329                dp = dp.wrapping_offset(ds);
330                ap = ap.wrapping_offset(a_s);
331                bp = bp.wrapping_offset(b_s);
332            }
333        });
334    }
335}
336
337/// Binary multiplication inner loop for identity element ops.
338trait MulOutput<D: Copy + 'static>: Copy + Send + Sync + 'static {
339    type Slot: Copy + Send + Sync + 'static;
340
341    unsafe fn write(dst: *mut Self::Slot, value: D);
342
343    unsafe fn try_contiguous<A: 'static, B: 'static>(
344        dst: *mut Self::Slot,
345        len: usize,
346        a: &[A],
347        b: &[B],
348    ) -> bool;
349}
350
351#[derive(Clone, Copy)]
352struct InitializedOutput;
353
354impl<D: Copy + Send + Sync + 'static> MulOutput<D> for InitializedOutput {
355    type Slot = D;
356
357    #[inline(always)]
358    unsafe fn write(dst: *mut D, value: D) {
359        unsafe { dst.write(value) };
360    }
361
362    #[inline(always)]
363    unsafe fn try_contiguous<A: 'static, B: 'static>(
364        dst: *mut D,
365        len: usize,
366        a: &[A],
367        b: &[B],
368    ) -> bool {
369        unsafe { simd::try_mul_contiguous_ptr(dst, len, a, b) }
370    }
371}
372
373#[derive(Clone, Copy)]
374struct UninitializedOutput;
375
376impl<D: Copy + Send + Sync + 'static> MulOutput<D> for UninitializedOutput {
377    type Slot = MaybeUninit<D>;
378
379    #[inline(always)]
380    unsafe fn write(dst: *mut MaybeUninit<D>, value: D) {
381        unsafe { dst.write(MaybeUninit::new(value)) };
382    }
383
384    #[inline(always)]
385    unsafe fn try_contiguous<A: 'static, B: 'static>(
386        dst: *mut MaybeUninit<D>,
387        len: usize,
388        a: &[A],
389        b: &[B],
390    ) -> bool {
391        // SIMD stores write complete values directly through raw pointers; no
392        // initialized reference is formed over the uninitialized destination.
393        unsafe { simd::try_mul_contiguous_ptr(dst.cast::<D>(), len, a, b) }
394    }
395}
396
397#[inline(always)]
398fn multiply_value<A, B, D>(lhs: A, rhs: B) -> D
399where
400    A: Copy + Mul<B, Output = D> + 'static,
401    B: Copy + 'static,
402    D: Copy + 'static,
403{
404    use core::any::TypeId;
405
406    if TypeId::of::<A>() == TypeId::of::<i32>()
407        && TypeId::of::<B>() == TypeId::of::<i32>()
408        && TypeId::of::<D>() == TypeId::of::<i32>()
409    {
410        // The exact TypeId checks prove identical layout and validity.
411        let lhs = unsafe { *(&lhs as *const A).cast::<i32>() };
412        let rhs = unsafe { *(&rhs as *const B).cast::<i32>() };
413        let value = lhs.wrapping_mul(rhs);
414        return unsafe { core::mem::transmute_copy(&value) };
415    }
416    if TypeId::of::<A>() == TypeId::of::<i64>()
417        && TypeId::of::<B>() == TypeId::of::<i64>()
418        && TypeId::of::<D>() == TypeId::of::<i64>()
419    {
420        // The exact TypeId checks prove identical layout and validity.
421        let lhs = unsafe { *(&lhs as *const A).cast::<i64>() };
422        let rhs = unsafe { *(&rhs as *const B).cast::<i64>() };
423        let value = lhs.wrapping_mul(rhs);
424        return unsafe { core::mem::transmute_copy(&value) };
425    }
426    lhs * rhs
427}
428
429#[inline(always)]
430unsafe fn inner_loop_mul2<
431    O: MulOutput<D>,
432    D: Copy + 'static,
433    A: Copy + Mul<B, Output = D> + 'static,
434    B: Copy + 'static,
435>(
436    dp: *mut O::Slot,
437    ds: isize,
438    ap: *const A,
439    a_s: isize,
440    bp: *const B,
441    b_s: isize,
442    len: usize,
443) {
444    if ds == 1 && a_s == 1 && b_s == 1 {
445        let src_a = std::slice::from_raw_parts(ap, len);
446        let src_b = std::slice::from_raw_parts(bp, len);
447        if len >= 64 && O::try_contiguous(dp, len, src_a, src_b) {
448            return;
449        }
450        simd::dispatch_if_large(len, || {
451            for i in 0..len {
452                O::write(dp.add(i), multiply_value(src_a[i], src_b[i]));
453            }
454        });
455    } else if ds == 1 && a_s == 1 && b_s == 0 {
456        let src_a = std::slice::from_raw_parts(ap, len);
457        let b = *bp;
458        simd::dispatch_if_large(len, || {
459            // The stride-dispatch branches below all walk `0..len` against the
460            // destination pointer; keep this one in the same shape.
461            #[allow(clippy::needless_range_loop)]
462            for i in 0..len {
463                O::write(dp.add(i), multiply_value(src_a[i], b));
464            }
465        });
466    } else if ds == 1 && a_s == 0 && b_s == 1 {
467        let a = *ap;
468        let src_b = std::slice::from_raw_parts(bp, len);
469        simd::dispatch_if_large(len, || {
470            // Same shape as the branch above, mirrored: the walk is over the
471            // destination pointer.
472            #[allow(clippy::needless_range_loop)]
473            for i in 0..len {
474                O::write(dp.add(i), multiply_value(a, src_b[i]));
475            }
476        });
477    } else if ds == 1 && a_s == 0 && b_s == 0 {
478        let a = *ap;
479        let b = *bp;
480        simd::dispatch_if_large(len, || {
481            for i in 0..len {
482                O::write(dp.add(i), multiply_value(a, b));
483            }
484        });
485    } else if ds == 1 && b_s == 0 {
486        let b = *bp;
487        let mut ap = ap;
488        simd::dispatch_if_large(len, || {
489            for i in 0..len {
490                O::write(dp.add(i), multiply_value(*ap, b));
491                ap = ap.wrapping_offset(a_s);
492            }
493        });
494    } else if ds == 1 && a_s == 0 {
495        let a = *ap;
496        let mut bp = bp;
497        simd::dispatch_if_large(len, || {
498            for i in 0..len {
499                O::write(dp.add(i), multiply_value(a, *bp));
500                bp = bp.wrapping_offset(b_s);
501            }
502        });
503    } else {
504        let mut dp = dp;
505        let mut ap = ap;
506        let mut bp = bp;
507        simd::dispatch_if_large(len, || {
508            for _ in 0..len {
509                O::write(dp, multiply_value(*ap, *bp));
510                dp = dp.wrapping_offset(ds);
511                ap = ap.wrapping_offset(a_s);
512                bp = bp.wrapping_offset(b_s);
513            }
514        });
515    }
516}
517
518#[derive(Clone, Debug, Eq, PartialEq)]
519struct ContiguousMulRangePlan {
520    axis_order: AxisVec<usize>,
521    inner_len: usize,
522    inner_axis_count: usize,
523    row_len: usize,
524    outer_axis_start: usize,
525    fast_axis: usize,
526    a_fast_stride: isize,
527    b_fast_stride: isize,
528    a_row_stride: isize,
529    b_row_stride: isize,
530}
531
532impl ContiguousMulRangePlan {
533    /// Whether running the plan row by row, without cache blocking, is sound
534    /// for performance.
535    ///
536    /// The destination is walked contiguously, so a unit-stride or broadcast
537    /// (stride 0) input streams along with it. An input read at any other
538    /// stride along the fast axis (a transposed operand) is swept across the
539    /// whole fast axis once per row, touching a new cache line per element and
540    /// again on the next row; the blocked kernel tiles that access. The one
541    /// exception is the transposed-scalar tile, which blocks itself.
542    fn walks_inputs_without_blocking(&self) -> bool {
543        let streams = |stride: isize| stride == 0 || stride == 1;
544        if streams(self.a_fast_stride) && streams(self.b_fast_stride) {
545            return true;
546        }
547        if transposed_scalar_tile_kind(self).is_some() {
548            return true;
549        }
550        false
551    }
552}
553
554#[derive(Clone, Copy, Debug, Eq, PartialEq)]
555enum TransposedScalarTileKind {
556    RhsScalar,
557    LhsScalar,
558}
559
560fn transposed_scalar_tile_kind(plan: &ContiguousMulRangePlan) -> Option<TransposedScalarTileKind> {
561    if plan.inner_len > TRANSPOSED_SCALAR_TILE_MAX_INNER_ROW_ASPECT.saturating_mul(plan.row_len) {
562        return None;
563    }
564    let row_len = isize::try_from(plan.row_len).ok()?;
565    if plan.a_fast_stride == row_len
566        && plan.a_row_stride == 1
567        && plan.b_fast_stride == 0
568        && plan.b_row_stride == 0
569    {
570        return Some(TransposedScalarTileKind::RhsScalar);
571    }
572
573    if plan.b_fast_stride == row_len
574        && plan.b_row_stride == 1
575        && plan.a_fast_stride == 0
576        && plan.a_row_stride == 0
577    {
578        return Some(TransposedScalarTileKind::LhsScalar);
579    }
580
581    None
582}
583
584fn compact_axis_order(dims: &[usize], strides: &[isize]) -> Option<AxisVec<usize>> {
585    if dims.len() != strides.len() {
586        return None;
587    }
588
589    let mut active = AxisVec::<usize>::new();
590    let mut inactive = AxisVec::<usize>::new();
591    for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
592        if stride < 0 {
593            return None;
594        }
595        if dim > 1 {
596            active.push(axis);
597        } else {
598            inactive.push(axis);
599        }
600    }
601
602    active.sort_by(|&lhs, &rhs| strides[lhs].cmp(&strides[rhs]).then_with(|| lhs.cmp(&rhs)));
603
604    let mut expected = 1isize;
605    for &axis in &active {
606        if strides[axis] != expected {
607            return None;
608        }
609        expected = expected.saturating_mul(dims[axis] as isize);
610    }
611
612    active.extend(inactive);
613    Some(active)
614}
615
616fn can_fuse_contiguous_range_axis(dim: usize, prev_stride: isize, next_stride: isize) -> bool {
617    dim <= 1 || (prev_stride == 0 && next_stride == 0) || next_stride == prev_stride * dim as isize
618}
619
620fn contiguous_mul_range_plan(
621    dims: &[usize],
622    dst_strides: &[isize],
623    a_strides: &[isize],
624    b_strides: &[isize],
625) -> Option<ContiguousMulRangePlan> {
626    let axis_order = compact_axis_order(dims, dst_strides)?;
627    if dims.is_empty() {
628        return Some(ContiguousMulRangePlan {
629            axis_order,
630            inner_len: 1,
631            inner_axis_count: 0,
632            row_len: 1,
633            outer_axis_start: 0,
634            fast_axis: 0,
635            a_fast_stride: 0,
636            b_fast_stride: 0,
637            a_row_stride: 0,
638            b_row_stride: 0,
639        });
640    }
641
642    let first_pos = axis_order
643        .iter()
644        .position(|&axis| dims[axis] > 1)
645        .unwrap_or(0);
646    let first_axis = axis_order[first_pos];
647    let mut inner_len = dims[first_axis].max(1);
648    let mut inner_axis_count = first_pos + 1;
649    let mut prev_axis = first_axis;
650
651    for &axis in axis_order.iter().skip(first_pos + 1) {
652        if can_fuse_contiguous_range_axis(
653            dims[prev_axis],
654            dst_strides[prev_axis],
655            dst_strides[axis],
656        ) && can_fuse_contiguous_range_axis(
657            dims[prev_axis],
658            a_strides[prev_axis],
659            a_strides[axis],
660        ) && can_fuse_contiguous_range_axis(
661            dims[prev_axis],
662            b_strides[prev_axis],
663            b_strides[axis],
664        ) {
665            inner_len = inner_len.checked_mul(dims[axis].max(1))?;
666            inner_axis_count += 1;
667            prev_axis = axis;
668        } else {
669            break;
670        }
671    }
672
673    let row = axis_order
674        .iter()
675        .enumerate()
676        .skip(inner_axis_count)
677        .find(|&(_, &axis)| dims[axis] > 1);
678    let (row_len, outer_axis_start, a_row_stride, b_row_stride) =
679        if let Some((row_pos, &row_axis)) = row {
680            (
681                dims[row_axis],
682                row_pos + 1,
683                a_strides[row_axis],
684                b_strides[row_axis],
685            )
686        } else {
687            (1, axis_order.len(), 0, 0)
688        };
689
690    Some(ContiguousMulRangePlan {
691        axis_order,
692        inner_len,
693        inner_axis_count,
694        row_len,
695        outer_axis_start,
696        fast_axis: first_axis,
697        a_fast_stride: a_strides[first_axis],
698        b_fast_stride: b_strides[first_axis],
699        a_row_stride,
700        b_row_stride,
701    })
702}
703
704struct ContiguousMulOuterCursor<'a> {
705    dims: &'a [usize],
706    a_strides: &'a [isize],
707    b_strides: &'a [isize],
708    axes: AxisVec<usize>,
709    coords: AxisVec<usize>,
710    a_offset: isize,
711    b_offset: isize,
712}
713
714impl<'a> ContiguousMulOuterCursor<'a> {
715    fn new(
716        dims: &'a [usize],
717        a_strides: &'a [isize],
718        b_strides: &'a [isize],
719        plan: &ContiguousMulRangePlan,
720        outer_group: usize,
721    ) -> Self {
722        let axes: AxisVec<usize> = plan
723            .axis_order
724            .iter()
725            .skip(plan.outer_axis_start)
726            .copied()
727            .collect();
728        let mut coords = AxisVec::<usize>::with_capacity(axes.len());
729        let mut rem = outer_group;
730        let mut a_offset = 0isize;
731        let mut b_offset = 0isize;
732
733        for &axis in &axes {
734            let dim = dims[axis].max(1);
735            let coord = rem % dim;
736            rem /= dim;
737            coords.push(coord);
738            a_offset += coord as isize * a_strides[axis];
739            b_offset += coord as isize * b_strides[axis];
740        }
741
742        Self {
743            dims,
744            a_strides,
745            b_strides,
746            axes,
747            coords,
748            a_offset,
749            b_offset,
750        }
751    }
752
753    fn advance(&mut self) {
754        for (i, &axis) in self.axes.iter().enumerate() {
755            let dim = self.dims[axis].max(1);
756            if dim <= 1 {
757                continue;
758            }
759
760            // Advance only when another position along this axis follows,
761            // and rewind from the last one, so offsets stay reachable.
762            if self.coords[i] + 1 < dim {
763                self.coords[i] += 1;
764                self.a_offset += self.a_strides[axis];
765                self.b_offset += self.b_strides[axis];
766                break;
767            }
768
769            let last = (dim - 1) as isize;
770            self.coords[i] = 0;
771            self.a_offset -= last * self.a_strides[axis];
772            self.b_offset -= last * self.b_strides[axis];
773        }
774    }
775}
776
777// Row block: both operand bases with their strides, the plan, the start index and
778// the element count.
779#[allow(clippy::too_many_arguments)]
780#[inline(always)]
781unsafe fn run_contiguous_mul_row_block<
782    O: MulOutput<D>,
783    D: Copy + 'static,
784    A: Copy + Mul<B, Output = D> + 'static,
785    B: Copy + 'static,
786>(
787    dst_ptr: *mut O::Slot,
788    a_ptr: *const A,
789    b_ptr: *const B,
790    plan: &ContiguousMulRangePlan,
791    base_index: usize,
792    total: usize,
793    base_a_offset: isize,
794    base_b_offset: isize,
795) {
796    let inner_len = plan.inner_len.max(1);
797    let row_len = plan.row_len.max(1);
798    let block_len = inner_len.saturating_mul(row_len);
799
800    if total.saturating_sub(base_index) >= block_len {
801        let tiled = match transposed_scalar_tile_kind(plan) {
802            Some(TransposedScalarTileKind::RhsScalar) => {
803                simd::try_mul_transposed_scalar_rhs_2d::<D, A, B>(
804                    dst_ptr.add(base_index).cast::<D>(),
805                    a_ptr.offset(base_a_offset),
806                    b_ptr.offset(base_b_offset),
807                    inner_len,
808                    row_len,
809                    plan.a_fast_stride,
810                    plan.a_row_stride,
811                )
812            }
813            Some(TransposedScalarTileKind::LhsScalar) => {
814                simd::try_mul_transposed_scalar_lhs_2d::<D, A, B>(
815                    dst_ptr.add(base_index).cast::<D>(),
816                    a_ptr.offset(base_a_offset),
817                    b_ptr.offset(base_b_offset),
818                    inner_len,
819                    row_len,
820                    plan.b_fast_stride,
821                    plan.b_row_stride,
822                )
823            }
824            None => false,
825        };
826        if tiled {
827            return;
828        }
829    }
830
831    let mut index = base_index;
832    let mut a_offset = base_a_offset;
833    let mut b_offset = base_b_offset;
834
835    for _ in 0..row_len {
836        if index >= total {
837            break;
838        }
839        let len = inner_len.min(total - index);
840        inner_loop_mul2::<O, D, A, B>(
841            dst_ptr.add(index),
842            1,
843            a_ptr.offset(a_offset),
844            plan.a_fast_stride,
845            b_ptr.offset(b_offset),
846            plan.b_fast_stride,
847            len,
848        );
849        index += inner_len;
850        a_offset += plan.a_row_stride;
851        b_offset += plan.b_row_stride;
852    }
853}
854
855fn strided_offset_for_contiguous_linear_index(
856    dims: &[usize],
857    strides: &[isize],
858    axis_order: &[usize],
859    mut index: usize,
860) -> isize {
861    let mut offset = 0isize;
862    for &axis in axis_order {
863        let dim = dims[axis];
864        if dim == 0 {
865            return 0;
866        }
867        let coord = index % dim;
868        index /= dim;
869        offset += coord as isize * strides[axis];
870    }
871    offset
872}
873
874// Raw parts entry for the whole contiguous product: the three base pointers, the
875// dims and strides of both operands and the executor. The argument list is wide
876// because this function splits the work itself (`parallel_for_each` below) from
877// exactly these values; nothing calls it once per worker range.
878#[allow(clippy::too_many_arguments)]
879fn try_contiguous_range_mul<
880    O: MulOutput<D>,
881    D: Copy + Send + Sync + 'static,
882    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
883    B: Copy + Send + Sync + 'static,
884>(
885    dst_ptr: *mut O::Slot,
886    dims: &[usize],
887    dst_strides: &[isize],
888    a_ptr: *const A,
889    a_strides: &[isize],
890    b_ptr: *const B,
891    b_strides: &[isize],
892    exec: Exec<'_>,
893) -> bool {
894    // An overflowing element count falls back to the general path, which
895    // reports it as `OffsetOverflow`.
896    let Ok(total) = total_len(dims) else {
897        return false;
898    };
899    if total == 0 {
900        return true;
901    }
902    // One flat section over the chunks: the width is the token's decision.
903    let exec = for_len(exec, KernelClass::Map, total);
904    if exec.is_serial() {
905        return false;
906    }
907
908    let Some(plan) = contiguous_mul_range_plan(dims, dst_strides, a_strides, b_strides) else {
909        return false;
910    };
911    if !plan.walks_inputs_without_blocking() {
912        return false;
913    }
914
915    let inner_len = plan.inner_len.max(1);
916    let row_len = plan.row_len.max(1);
917    let block_len = inner_len.saturating_mul(row_len).max(1);
918    let outer_groups = total.div_ceil(block_len);
919
920    let nthreads = exec.budget().get();
921    if nthreads > 1 {
922        use crate::threading::{parallel_for_each, SendPtr};
923
924        let dst = SendPtr(dst_ptr);
925        let a = SendPtr(a_ptr as *mut A);
926        let b = SendPtr(b_ptr as *mut B);
927
928        if outer_groups < nthreads {
929            let chunk_len = total.div_ceil(nthreads);
930            let nchunks = total.div_ceil(chunk_len);
931
932            parallel_for_each(0..nchunks, exec, &|chunks| {
933                for chunk in chunks {
934                    let start = chunk * chunk_len;
935                    let end = (start + chunk_len).min(total);
936                    let mut index = start;
937
938                    while index < end {
939                        let in_inner = index % inner_len;
940                        let len = (inner_len - in_inner).min(end - index);
941                        let a_offset = strided_offset_for_contiguous_linear_index(
942                            dims,
943                            a_strides,
944                            &plan.axis_order,
945                            index,
946                        );
947                        let b_offset = strided_offset_for_contiguous_linear_index(
948                            dims,
949                            b_strides,
950                            &plan.axis_order,
951                            index,
952                        );
953
954                        unsafe {
955                            inner_loop_mul2::<O, D, A, B>(
956                                dst.as_ptr().add(index),
957                                1,
958                                a.as_const().offset(a_offset),
959                                plan.a_fast_stride,
960                                b.as_const().offset(b_offset),
961                                plan.b_fast_stride,
962                                len,
963                            );
964                        }
965                        index += len;
966                    }
967                }
968            });
969
970            return true;
971        }
972
973        let groups_per_chunk = outer_groups.div_ceil(nthreads);
974        let nchunks = outer_groups.div_ceil(groups_per_chunk);
975
976        parallel_for_each(0..nchunks, exec, &|chunks| {
977            for chunk in chunks {
978                let group_start = chunk * groups_per_chunk;
979                let group_end = (group_start + groups_per_chunk).min(outer_groups);
980                let mut cursor =
981                    ContiguousMulOuterCursor::new(dims, a_strides, b_strides, &plan, group_start);
982
983                for group in group_start..group_end {
984                    let index = group * block_len;
985                    unsafe {
986                        run_contiguous_mul_row_block::<O, D, A, B>(
987                            dst.as_ptr(),
988                            a.as_const(),
989                            b.as_const(),
990                            &plan,
991                            index,
992                            total,
993                            cursor.a_offset,
994                            cursor.b_offset,
995                        );
996                    }
997                    cursor.advance();
998                }
999            }
1000        });
1001
1002        true
1003    } else {
1004        run_contiguous_range_mul_single_thread::<O, D, A, B>(
1005            dst_ptr,
1006            dims,
1007            a_ptr,
1008            a_strides,
1009            b_ptr,
1010            b_strides,
1011            &plan,
1012            total,
1013            block_len,
1014            outer_groups,
1015        )
1016    }
1017}
1018
1019// Worker-range leaf of the contiguous multiply: both operand bases and stride
1020// tables, the plan, the extents and the range.
1021#[allow(clippy::too_many_arguments)]
1022fn run_contiguous_range_mul_single_thread<
1023    O: MulOutput<D>,
1024    D: Copy + Send + Sync + 'static,
1025    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
1026    B: Copy + Send + Sync + 'static,
1027>(
1028    dst_ptr: *mut O::Slot,
1029    dims: &[usize],
1030    a_ptr: *const A,
1031    a_strides: &[isize],
1032    b_ptr: *const B,
1033    b_strides: &[isize],
1034    plan: &ContiguousMulRangePlan,
1035    total: usize,
1036    block_len: usize,
1037    outer_groups: usize,
1038) -> bool {
1039    let mut cursor = ContiguousMulOuterCursor::new(dims, a_strides, b_strides, plan, 0);
1040    for group in 0..outer_groups {
1041        let index = group * block_len;
1042        unsafe {
1043            run_contiguous_mul_row_block::<O, D, A, B>(
1044                dst_ptr,
1045                a_ptr,
1046                b_ptr,
1047                plan,
1048                index,
1049                total,
1050                cursor.a_offset,
1051                cursor.b_offset,
1052            );
1053        }
1054        cursor.advance();
1055    }
1056    true
1057}
1058
1059/// Ternary inner loop: `dest[i] = f(a[i], b[i], c[i])`.
1060// Three operand pointers with their strides, the destination, the length and the
1061// element op: the call's inputs, not a block's.
1062#[allow(clippy::too_many_arguments)]
1063#[inline(always)]
1064unsafe fn inner_loop_map3<
1065    D: Copy,
1066    A: Copy,
1067    B: Copy,
1068    C: Copy,
1069    OpA: ElementOp<A>,
1070    OpB: ElementOp<B>,
1071    OpC: ElementOp<C>,
1072>(
1073    dp: *mut D,
1074    ds: isize,
1075    ap: *const A,
1076    a_s: isize,
1077    bp: *const B,
1078    b_s: isize,
1079    cp: *const C,
1080    c_s: isize,
1081    len: usize,
1082    f: &impl Fn(A, B, C) -> D,
1083) {
1084    if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 {
1085        let src_a = std::slice::from_raw_parts(ap, len);
1086        let src_b = std::slice::from_raw_parts(bp, len);
1087        let src_c = std::slice::from_raw_parts(cp, len);
1088        let dst = std::slice::from_raw_parts_mut(dp, len);
1089        simd::dispatch_if_large(len, || {
1090            for (((d, &a), &b), &c) in dst.iter_mut().zip(src_a).zip(src_b).zip(src_c) {
1091                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
1092            }
1093        });
1094    } else {
1095        let mut dp = dp;
1096        let mut ap = ap;
1097        let mut bp = bp;
1098        let mut cp = cp;
1099        simd::dispatch_if_large(len, || {
1100            for _ in 0..len {
1101                *dp = f(OpA::apply(*ap), OpB::apply(*bp), OpC::apply(*cp));
1102                dp = dp.wrapping_offset(ds);
1103                ap = ap.wrapping_offset(a_s);
1104                bp = bp.wrapping_offset(b_s);
1105                cp = cp.wrapping_offset(c_s);
1106            }
1107        });
1108    }
1109}
1110
1111/// Quaternary inner loop: `dest[i] = f(a[i], b[i], c[i], e[i])`.
1112// Four operand pointers with their strides, the destination, the length and the
1113// element op: the call's inputs, not a block's.
1114#[allow(clippy::too_many_arguments)]
1115#[inline(always)]
1116unsafe fn inner_loop_map4<
1117    D: Copy,
1118    A: Copy,
1119    B: Copy,
1120    C: Copy,
1121    E: Copy,
1122    OpA: ElementOp<A>,
1123    OpB: ElementOp<B>,
1124    OpC: ElementOp<C>,
1125    OpE: ElementOp<E>,
1126>(
1127    dp: *mut D,
1128    ds: isize,
1129    ap: *const A,
1130    a_s: isize,
1131    bp: *const B,
1132    b_s: isize,
1133    cp: *const C,
1134    c_s: isize,
1135    ep: *const E,
1136    e_s: isize,
1137    len: usize,
1138    f: &impl Fn(A, B, C, E) -> D,
1139) {
1140    if ds == 1 && a_s == 1 && b_s == 1 && c_s == 1 && e_s == 1 {
1141        let src_a = std::slice::from_raw_parts(ap, len);
1142        let src_b = std::slice::from_raw_parts(bp, len);
1143        let src_c = std::slice::from_raw_parts(cp, len);
1144        let src_e = std::slice::from_raw_parts(ep, len);
1145        let dst = std::slice::from_raw_parts_mut(dp, len);
1146        // Zipped iterators keep the loop free of bounds checks, which keeps the
1147        // dispatched closure small enough for pulp's target-feature function to
1148        // inline (tensor4all/strided-rs#340).
1149        simd::dispatch_if_large(len, || {
1150            for ((((d, &a), &b), &c), &e) in
1151                dst.iter_mut().zip(src_a).zip(src_b).zip(src_c).zip(src_e)
1152            {
1153                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
1154            }
1155        });
1156    } else {
1157        let mut dp = dp;
1158        let mut ap = ap;
1159        let mut bp = bp;
1160        let mut cp = cp;
1161        let mut ep = ep;
1162        simd::dispatch_if_large(len, || {
1163            for _ in 0..len {
1164                *dp = f(
1165                    OpA::apply(*ap),
1166                    OpB::apply(*bp),
1167                    OpC::apply(*cp),
1168                    OpE::apply(*ep),
1169                );
1170                dp = dp.wrapping_offset(ds);
1171                ap = ap.wrapping_offset(a_s);
1172                bp = bp.wrapping_offset(b_s);
1173                cp = cp.wrapping_offset(c_s);
1174                ep = ep.wrapping_offset(e_s);
1175            }
1176        });
1177    }
1178}
1179
1180/// An element function that carries its [`KernelClass`].
1181///
1182/// [`map_into`] and its siblings gate their parallel fork on the serial time of the
1183/// region, which is the element count times the throughput of the function's class.
1184/// A plain closure or function item is a [`KernelClass::Map`] (a few arithmetic
1185/// operations per element) through the blanket implementation, so existing callers
1186/// are unchanged. A caller whose function is expensive declares it with
1187/// [`Classed`], usually through [`KernelClass::with`], so a region too short to pay
1188/// for a fork as arithmetic is still forked when each element costs a libm call.
1189///
1190/// The trait is implemented for every `Fn(A) -> D + Sync` and for [`Classed`]; it is
1191/// not meant to be implemented by hand.
1192///
1193/// # Examples
1194/// ```
1195/// use strided_basic::execution::KernelClass;
1196/// use strided_basic::{map_into, Exec, StridedView, StridedViewMut};
1197///
1198/// let src = vec![0.0f64, 1.0, 2.0];
1199/// let mut out = vec![0.0f64; 3];
1200/// let mut dest = StridedViewMut::new(&mut out, &[3], &[1], 0).unwrap();
1201/// let source = StridedView::<f64>::new(&src, &[3], &[1], 0).unwrap();
1202/// // `exp` costs about as much as ten arithmetic operations: say so.
1203/// let exp = KernelClass::Transcendental.with(|x: f64| x.exp());
1204/// map_into(&mut dest, &source, exp, Exec::serial()).unwrap();
1205/// assert_eq!(out[0], 1.0);
1206/// ```
1207pub trait MapFn<A, D>: Sync {
1208    /// Apply the function to one element.
1209    fn call(&self, a: A) -> D;
1210
1211    /// Serial cost class the parallel gate charges per element; `Map` by default.
1212    fn class(&self) -> KernelClass {
1213        KernelClass::Map
1214    }
1215}
1216
1217impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for F {
1218    #[inline(always)]
1219    fn call(&self, a: A) -> D {
1220        self(a)
1221    }
1222}
1223
1224/// An element function together with the [`KernelClass`] it is charged as.
1225///
1226/// Build one with [`Classed::new`] or [`KernelClass::with`]; see [`MapFn`].
1227#[derive(Debug, Clone, Copy)]
1228pub struct Classed<F> {
1229    /// The element function.
1230    pub f: F,
1231    /// The class the parallel gate charges per element.
1232    pub class: KernelClass,
1233}
1234
1235impl<F> Classed<F> {
1236    /// Declare `f` as a function of cost class `class`.
1237    pub const fn new(class: KernelClass, f: F) -> Self {
1238        Self { f, class }
1239    }
1240}
1241
1242impl KernelClass {
1243    /// Declare an element function of this class, for [`map_into`] and its siblings.
1244    pub const fn with<F>(self, f: F) -> Classed<F> {
1245        Classed::new(self, f)
1246    }
1247}
1248
1249impl<A, D, F: Fn(A) -> D + Sync> MapFn<A, D> for Classed<F> {
1250    #[inline(always)]
1251    fn call(&self, a: A) -> D {
1252        (self.f)(a)
1253    }
1254
1255    fn class(&self) -> KernelClass {
1256        self.class
1257    }
1258}
1259
1260/// Apply a function element-wise from source to destination.
1261///
1262/// The element operation `Op` is applied lazily when reading from `src`.
1263/// Source and destination may have different element types.
1264///
1265/// `f` is any `Fn(A) -> D + Sync`, gated as [`KernelClass::Map`], or a function declared
1266/// with a class of its own (see [`MapFn`]), so an expensive element function forks sooner.
1267///
1268/// # Errors
1269///
1270/// Returns [`StridedError::RankMismatch`] or
1271/// [`StridedError::ShapeMismatch`] when the views have different ranks or
1272/// extents, and [`StridedError::NonInjectiveOutputLayout`] when the
1273/// destination layout cannot be proven representable and injective: two
1274/// logical indices name one offset, or the element count or address span is
1275/// too large to represent. Nothing is written unless validation succeeds.
1276pub fn map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1277    dest: &mut StridedViewMut<D>,
1278    src: &StridedView<A, Op>,
1279    f: impl MapFn<A, D>,
1280    exec: Exec<'_>,
1281) -> Result<()> {
1282    let class = f.class();
1283    map_parts_into::<D, A, Op, NoKernel>(
1284        dest.as_mut_ptr(),
1285        dest.dims(),
1286        dest.strides(),
1287        src.ptr(),
1288        src.dims(),
1289        src.strides(),
1290        |a| f.call(a),
1291        &NoKernel,
1292        class,
1293        exec,
1294    )
1295}
1296
1297/// `map_into` for a pure copy: the same traversal, gated as [`KernelClass::Copy`].
1298///
1299/// A copy has no arithmetic per element, so a fork over it pays later than over
1300/// a general element map; asking as `Map` forked it too early.
1301pub(crate) fn copy_map_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1302    dest: &mut StridedViewMut<D>,
1303    src: &StridedView<A, Op>,
1304    f: impl Fn(A) -> D + Sync,
1305    exec: Exec<'_>,
1306) -> Result<()> {
1307    map_parts_into::<D, A, Op, NoKernel>(
1308        dest.as_mut_ptr(),
1309        dest.dims(),
1310        dest.strides(),
1311        src.ptr(),
1312        src.dims(),
1313        src.strides(),
1314        f,
1315        &NoKernel,
1316        KernelClass::Copy,
1317        exec,
1318    )
1319}
1320
1321pub(crate) fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1322    dest: &mut StridedViewMut<D>,
1323    src: &StridedView<A, Op>,
1324    f: impl MapFn<A, D>,
1325    validated: ValidatedDestinationLayout,
1326    exec: Exec<'_>,
1327) -> Result<()> {
1328    ensure_same_shape(dest.dims(), src.dims())?;
1329    let class = f.class();
1330    map_parts_into_validated::<D, A, Op, NoKernel>(
1331        dest.as_mut_ptr(),
1332        dest.dims(),
1333        dest.strides(),
1334        src.ptr(),
1335        src.strides(),
1336        |a| f.call(a),
1337        &NoKernel,
1338        validated,
1339        class,
1340        exec,
1341    )
1342}
1343
1344/// Apply an element-wise function into a potentially uninitialized destination, handing a
1345/// vectorized kernel the flat runs it can take.
1346///
1347/// `kernel` is offered a flat, unit-stride run whenever this crate has one to give: the whole
1348/// buffer when the two views share a contiguous layout and the operation runs sequentially, and
1349/// one block at a time on the parallel path. It must write every element of the run or write
1350/// nothing — returning `false` makes that run fall back to `scalar`, element by element. Runs whose
1351/// strides are not both 1 never reach it. It is called from whichever tasks the budget of
1352/// `exec` selected, so it must be `Sync`.
1353///
1354/// The point of the entry point is that a caller which owns vector kernels — a SIMD math library,
1355/// say — reuses this crate's layout logic and this crate's threading decision instead of
1356/// rebuilding either. `scalar` and `kernel` must agree up to the caller's own error bound; this
1357/// crate only guarantees that every destination element is written exactly once.
1358///
1359/// `scalar` may carry a [`KernelClass`] (see [`MapFn`]); the gate charges the region as that
1360/// class, so a transcendental `scalar` forks sooner.
1361///
1362/// # Examples
1363/// ```
1364/// use std::mem::MaybeUninit;
1365/// use strided_basic::{map_into_uninit_with_kernel, Exec, StridedView, StridedViewMut};
1366///
1367/// let src = vec![1.0f64, 2.0, 3.0];
1368/// let mut out = vec![MaybeUninit::<f64>::uninit(); 3];
1369/// let mut dest = StridedViewMut::new(&mut out, &[3], &[1], 0).unwrap();
1370/// let source = StridedView::new(&src, &[3], &[1], 0).unwrap();
1371/// let kernel = |src: &[f64], dst: &mut [MaybeUninit<f64>]| {
1372///     for (slot, value) in dst.iter_mut().zip(src) {
1373///         slot.write(value * 2.0);
1374///     }
1375///     true
1376/// };
1377/// map_into_uninit_with_kernel(&mut dest, &source, kernel, |x| x * 2.0, Exec::serial()).unwrap();
1378/// // SAFETY: the kernel wrote every element.
1379/// let values: Vec<f64> = out.into_iter().map(|x| unsafe { x.assume_init() }).collect();
1380/// assert_eq!(values, [2.0, 4.0, 6.0]);
1381/// ```
1382///
1383/// # Errors
1384/// Returns [`StridedError::ShapeMismatch`] when the shapes differ and
1385/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout cannot be proven
1386/// injective.
1387pub fn map_into_uninit_with_kernel<T: Copy + Send + Sync>(
1388    dest: &mut StridedViewMut<MaybeUninit<T>>,
1389    src: &StridedView<T>,
1390    kernel: impl Fn(&[T], &mut [MaybeUninit<T>]) -> bool + Sync,
1391    scalar: impl MapFn<T, T>,
1392    exec: Exec<'_>,
1393) -> Result<()> {
1394    let class = scalar.class();
1395    map_parts_into::<MaybeUninit<T>, T, Identity, _>(
1396        dest.as_mut_ptr(),
1397        dest.dims(),
1398        dest.strides(),
1399        src.ptr(),
1400        src.dims(),
1401        src.strides(),
1402        |value| MaybeUninit::new(scalar.call(value)),
1403        &SliceKernel(kernel),
1404        class,
1405        exec,
1406    )
1407}
1408
1409/// Map a raw destination from a raw source without preparing a plan.
1410///
1411/// Only the raw-replay unit tests need this entry now; the crate's production
1412/// callers either go through the fused traversal (raw ops, `CopyPlan`) or
1413/// [`map_raw_into_validated`].
1414#[cfg(test)]
1415pub(crate) fn map_raw_into<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
1416    dest: &mut crate::RawStridedMut<'_, D>,
1417    src: &crate::RawStridedRef<'_, A>,
1418    f: impl Fn(A) -> D + Sync,
1419    exec: Exec<'_>,
1420) -> Result<()> {
1421    map_parts_into::<D, A, Op, NoKernel>(
1422        dest.as_mut_ptr(),
1423        dest.dims(),
1424        dest.strides(),
1425        src.ptr(),
1426        src.dims(),
1427        src.strides(),
1428        f,
1429        &NoKernel,
1430        KernelClass::Map,
1431        exec,
1432    )
1433}
1434
1435pub(crate) fn map_raw_into_validated<
1436    D: Copy + Send + Sync,
1437    A: Copy + Send + Sync,
1438    Op: ElementOp<A>,
1439>(
1440    dest: &mut crate::RawStridedMut<'_, D>,
1441    src: &crate::RawStridedRef<'_, A>,
1442    f: impl Fn(A) -> D + Sync,
1443    validated: ValidatedDestinationLayout,
1444    exec: Exec<'_>,
1445) -> Result<()> {
1446    ensure_same_shape(dest.dims(), src.dims())?;
1447    map_parts_into_validated::<D, A, Op, NoKernel>(
1448        dest.as_mut_ptr(),
1449        dest.dims(),
1450        dest.strides(),
1451        src.ptr(),
1452        src.strides(),
1453        f,
1454        &NoKernel,
1455        validated,
1456        KernelClass::Map,
1457        exec,
1458    )
1459}
1460
1461// Raw parts for the whole operation, not one block: this entry point checks the shapes and
1462// the destination layout, then forwards the same parts to `map_parts_into_validated`.
1463#[allow(clippy::too_many_arguments)]
1464fn map_parts_into<
1465    D: Copy + Send + Sync,
1466    A: Copy + Send + Sync,
1467    Op: ElementOp<A>,
1468    K: MapKernel<A, D> + Sync,
1469>(
1470    dst_ptr: *mut D,
1471    dst_dims: &[usize],
1472    dst_strides: &[isize],
1473    src_ptr: *const A,
1474    src_dims: &[usize],
1475    src_strides: &[isize],
1476    f: impl Fn(A) -> D + Sync,
1477    kernel: &K,
1478    class: KernelClass,
1479    exec: Exec<'_>,
1480) -> Result<()> {
1481    ensure_same_shape(dst_dims, src_dims)?;
1482    let validated = validate_destination_layout(dst_dims, dst_strides)?;
1483    map_parts_into_validated::<D, A, Op, K>(
1484        dst_ptr,
1485        dst_dims,
1486        dst_strides,
1487        src_ptr,
1488        src_strides,
1489        f,
1490        kernel,
1491        validated,
1492        class,
1493        exec,
1494    )
1495}
1496
1497#[allow(clippy::too_many_arguments)]
1498/// Apply `f` (with the element operation) over a fully contiguous pair.
1499///
1500/// Kept out of line, and out of the caller's dispatch frame, so that it cannot
1501/// change the code shape of the blocked kernels below; the operation is moved in
1502/// by value because a dispatched closure that only borrows it stays scalar
1503/// (tensor4all/strided-rs#301).
1504#[inline(never)]
1505fn map_contiguous<D: Copy, A: Copy, Op: ElementOp<A>>(
1506    dst: &mut [D],
1507    src: &[A],
1508    f: impl Fn(A) -> D,
1509) {
1510    simd::dispatch_if_large(dst.len(), move || {
1511        for i in 0..dst.len() {
1512            dst[i] = f(Op::apply(src[i]));
1513        }
1514    });
1515}
1516
1517// Raw parts, the element op, the kernel, the class and the token: the operands are the
1518// operation's, not a block's.
1519#[allow(clippy::too_many_arguments)]
1520fn map_parts_into_validated<
1521    D: Copy + Send + Sync,
1522    A: Copy + Send + Sync,
1523    Op: ElementOp<A>,
1524    K: MapKernel<A, D> + Sync,
1525>(
1526    dst_ptr: *mut D,
1527    dst_dims: &[usize],
1528    dst_strides: &[isize],
1529    src_ptr: *const A,
1530    src_strides: &[isize],
1531    f: impl Fn(A) -> D + Sync,
1532    kernel: &K,
1533    _validated: ValidatedDestinationLayout,
1534    class: KernelClass,
1535    exec: Exec<'_>,
1536) -> Result<()> {
1537    if sequential_contiguous_layout(class, dst_dims, &[dst_strides, src_strides], &exec)?.is_some()
1538    {
1539        let len = total_len(dst_dims)?;
1540        // SAFETY: the contiguous-layout check proves that `dst` and `src` address `len` elements in
1541        // the same order starting at their base pointer, so each is exactly a flat run of `len`
1542        // elements.
1543        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1544        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
1545        if kernel.apply(src, dst) {
1546            return Ok(());
1547        }
1548        map_contiguous::<D, A, Op>(dst, src, f);
1549        return Ok(());
1550    }
1551
1552    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
1553    let total = total_len(dst_dims)?;
1554    let (fused_dims, ordered_strides, plan) = pointwise_plan(
1555        dst_dims,
1556        &strides_list,
1557        Some(0),
1558        &[std::mem::size_of::<D>(), std::mem::size_of::<A>()],
1559        total,
1560    );
1561
1562    let total = total_len(&fused_dims)?;
1563    if is_parallel(class, total, &exec) {
1564        use crate::threading::SendPtr;
1565        let dst_send = SendPtr(dst_ptr);
1566        let src_send = SendPtr(src_ptr as *mut A);
1567
1568        let costs = compute_costs(&ordered_strides);
1569        let initial_offsets: crate::inline::OperandVec<isize> =
1570            strides_list.iter().map(|_| 0isize).collect();
1571        return mapreduce_threaded(
1572            &fused_dims,
1573            &plan.block,
1574            &ordered_strides,
1575            &initial_offsets,
1576            &costs,
1577            class,
1578            exec,
1579            &|dims, blocks, strides_list, offsets| {
1580                for_each_inner_block_with_offsets(
1581                    dims,
1582                    blocks,
1583                    strides_list,
1584                    offsets,
1585                    |offsets, len, strides| {
1586                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1587                        let sp = unsafe { src_send.as_const().offset(offsets[1]) };
1588                        unsafe {
1589                            inner_loop_map1::<D, A, Op, K>(
1590                                dp, strides[0], sp, strides[1], len, &f, kernel,
1591                            )
1592                        };
1593                        Ok(())
1594                    },
1595                )
1596            },
1597        );
1598    }
1599
1600    let initial_offsets: crate::inline::OperandVec<isize> =
1601        ordered_strides.iter().map(|_| 0isize).collect();
1602    for_each_inner_block_preordered(
1603        &fused_dims,
1604        &plan.block,
1605        &ordered_strides,
1606        &initial_offsets,
1607        |offsets, len, strides| {
1608            let dp = unsafe { dst_ptr.offset(offsets[0]) };
1609            let sp = unsafe { src_ptr.offset(offsets[1]) };
1610            unsafe {
1611                inner_loop_map1::<D, A, Op, K>(dp, strides[0], sp, strides[1], len, &f, kernel)
1612            };
1613            Ok(())
1614        },
1615    )
1616}
1617
1618/// Binary element-wise operation: `dest[i] = f(a[i], b[i])`.
1619///
1620/// Source operands `a` and `b` may have different element types from each other
1621/// and from `dest`. The closure `f` handles per-element type conversion.
1622///
1623/// # Errors
1624///
1625/// Returns [`StridedError::RankMismatch`] or
1626/// [`StridedError::ShapeMismatch`] when `dest` and either source view have
1627/// different ranks or extents, and
1628/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1629/// cannot be proven representable and injective: two logical indices name
1630/// one offset, or the element count or address span is too large to
1631/// represent. Nothing is written unless validation succeeds.
1632pub fn zip_map2_into<
1633    D: Copy + Send + Sync,
1634    A: Copy + Send + Sync,
1635    B: Copy + Send + Sync,
1636    OpA: ElementOp<A>,
1637    OpB: ElementOp<B>,
1638>(
1639    dest: &mut StridedViewMut<D>,
1640    a: &StridedView<A, OpA>,
1641    b: &StridedView<B, OpB>,
1642    f: impl Fn(A, B) -> D + Sync,
1643    exec: Exec<'_>,
1644) -> Result<()> {
1645    zip_map2_parts_into::<D, A, B, OpA, OpB>(
1646        dest.as_mut_ptr(),
1647        dest.dims(),
1648        dest.strides(),
1649        a.ptr(),
1650        a.dims(),
1651        a.strides(),
1652        b.ptr(),
1653        b.dims(),
1654        b.strides(),
1655        f,
1656        exec,
1657    )
1658}
1659
1660pub(crate) fn zip_map2_into_validated<
1661    D: Copy + Send + Sync,
1662    A: Copy + Send + Sync,
1663    B: Copy + Send + Sync,
1664    OpA: ElementOp<A>,
1665    OpB: ElementOp<B>,
1666>(
1667    dest: &mut StridedViewMut<D>,
1668    a: &StridedView<A, OpA>,
1669    b: &StridedView<B, OpB>,
1670    f: impl Fn(A, B) -> D + Sync,
1671    validated: ValidatedDestinationLayout,
1672    exec: Exec<'_>,
1673) -> Result<()> {
1674    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1675        dest.as_mut_ptr(),
1676        dest.dims(),
1677        dest.strides(),
1678        a.ptr(),
1679        a.strides(),
1680        b.ptr(),
1681        b.strides(),
1682        f,
1683        validated,
1684        exec,
1685    )
1686}
1687
1688/// Runtime comparison selected once before entering the element loop.
1689#[non_exhaustive]
1690#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1691pub enum CompareOp {
1692    Eq,
1693    Lt,
1694    Le,
1695    Gt,
1696    Ge,
1697}
1698
1699/// Compare two ordered views elementwise into a Boolean destination.
1700///
1701/// Unlike embedding a runtime operation match inside a [`zip_map2_into`]
1702/// closure, this entry point selects a fixed comparison before traversal.
1703///
1704/// # Errors
1705///
1706/// Returns [`StridedError::ShapeMismatch`] when the source and destination
1707/// shapes differ, or [`StridedError::NonInjectiveOutputLayout`] when distinct
1708/// logical destination elements may overlap.
1709pub fn compare_into<T, OpA, OpB>(
1710    dest: &mut StridedViewMut<bool>,
1711    a: &StridedView<T, OpA>,
1712    b: &StridedView<T, OpB>,
1713    op: CompareOp,
1714    exec: Exec<'_>,
1715) -> Result<()>
1716where
1717    T: Copy + Send + Sync + PartialOrd,
1718    OpA: ElementOp<T>,
1719    OpB: ElementOp<T>,
1720{
1721    match op {
1722        CompareOp::Eq => zip_map2_into(dest, a, b, |lhs, rhs| lhs == rhs, exec),
1723        CompareOp::Lt => zip_map2_into(dest, a, b, |lhs, rhs| lhs < rhs, exec),
1724        CompareOp::Le => zip_map2_into(dest, a, b, |lhs, rhs| lhs <= rhs, exec),
1725        CompareOp::Gt => zip_map2_into(dest, a, b, |lhs, rhs| lhs > rhs, exec),
1726        CompareOp::Ge => zip_map2_into(dest, a, b, |lhs, rhs| lhs >= rhs, exec),
1727    }
1728}
1729
1730/// Compare two views into a fully overwritten uninitialized Boolean output.
1731///
1732/// Dtype-independent shape, destination-injectivity, and reachable-byte
1733/// overlap validation completes before the first write. Safe Rust borrows
1734/// already prevent input/output aliasing; the explicit overlap check preserves
1735/// the contract for views produced through unsafe constructors.
1736///
1737/// `Ok(())` means every logical destination element is initialized. An error
1738/// occurs before writes. A panic during replay may leave a partially initialized
1739/// destination, which remains safe to drop as `MaybeUninit<bool>`.
1740///
1741/// # Errors
1742///
1743/// Returns [`StridedError::ShapeMismatch`] for unequal shapes,
1744/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
1745/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
1746/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
1747/// representable.
1748pub fn compare_into_uninit<T, OpA, OpB>(
1749    dest: &mut StridedViewMut<MaybeUninit<bool>>,
1750    a: &StridedView<T, OpA>,
1751    b: &StridedView<T, OpB>,
1752    op: CompareOp,
1753    exec: Exec<'_>,
1754) -> Result<()>
1755where
1756    T: Copy + Send + Sync + PartialOrd,
1757    OpA: ElementOp<T>,
1758    OpB: ElementOp<T>,
1759{
1760    ensure_same_shape(dest.dims(), a.dims())?;
1761    ensure_same_shape(dest.dims(), b.dims())?;
1762    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
1763    validate_typed_no_overlap(dest, a, 0)?;
1764    validate_typed_no_overlap(dest, b, 1)?;
1765    match op {
1766        CompareOp::Eq => zip_map2_into_validated(
1767            dest,
1768            a,
1769            b,
1770            |lhs, rhs| MaybeUninit::new(lhs == rhs),
1771            validated,
1772            exec,
1773        ),
1774        CompareOp::Lt => zip_map2_into_validated(
1775            dest,
1776            a,
1777            b,
1778            |lhs, rhs| MaybeUninit::new(lhs < rhs),
1779            validated,
1780            exec,
1781        ),
1782        CompareOp::Le => zip_map2_into_validated(
1783            dest,
1784            a,
1785            b,
1786            |lhs, rhs| MaybeUninit::new(lhs <= rhs),
1787            validated,
1788            exec,
1789        ),
1790        CompareOp::Gt => zip_map2_into_validated(
1791            dest,
1792            a,
1793            b,
1794            |lhs, rhs| MaybeUninit::new(lhs > rhs),
1795            validated,
1796            exec,
1797        ),
1798        CompareOp::Ge => zip_map2_into_validated(
1799            dest,
1800            a,
1801            b,
1802            |lhs, rhs| MaybeUninit::new(lhs >= rhs),
1803            validated,
1804            exec,
1805        ),
1806    }
1807}
1808
1809pub(crate) fn zip_map2_raw_into_validated<
1810    D: Copy + Send + Sync,
1811    A: Copy + Send + Sync,
1812    B: Copy + Send + Sync,
1813    OpA: ElementOp<A>,
1814    OpB: ElementOp<B>,
1815>(
1816    dest: &mut crate::RawStridedMut<'_, D>,
1817    a: &crate::RawStridedRef<'_, A>,
1818    b: &crate::RawStridedRef<'_, B>,
1819    f: impl Fn(A, B) -> D + Sync,
1820    validated: ValidatedDestinationLayout,
1821    exec: Exec<'_>,
1822) -> Result<()> {
1823    ensure_same_shape(dest.dims(), a.dims())?;
1824    ensure_same_shape(dest.dims(), b.dims())?;
1825    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1826        dest.as_mut_ptr(),
1827        dest.dims(),
1828        dest.strides(),
1829        a.ptr(),
1830        a.strides(),
1831        b.ptr(),
1832        b.strides(),
1833        f,
1834        validated,
1835        exec,
1836    )
1837}
1838
1839// Raw parts for the whole binary operation: the shape and destination-layout checks live
1840// here, and `zip_map2_parts_into_validated` below does the planning and dispatch.
1841#[allow(clippy::too_many_arguments)]
1842fn zip_map2_parts_into<
1843    D: Copy + Send + Sync,
1844    A: Copy + Send + Sync,
1845    B: Copy + Send + Sync,
1846    OpA: ElementOp<A>,
1847    OpB: ElementOp<B>,
1848>(
1849    dst_ptr: *mut D,
1850    dst_dims: &[usize],
1851    dst_strides: &[isize],
1852    a_ptr: *const A,
1853    a_dims: &[usize],
1854    a_strides: &[isize],
1855    b_ptr: *const B,
1856    b_dims: &[usize],
1857    b_strides: &[isize],
1858    f: impl Fn(A, B) -> D + Sync,
1859    exec: Exec<'_>,
1860) -> Result<()> {
1861    ensure_same_shape(dst_dims, a_dims)?;
1862    ensure_same_shape(dst_dims, b_dims)?;
1863    let validated = validate_destination_layout(dst_dims, dst_strides)?;
1864    zip_map2_parts_into_validated::<D, A, B, OpA, OpB>(
1865        dst_ptr,
1866        dst_dims,
1867        dst_strides,
1868        a_ptr,
1869        a_strides,
1870        b_ptr,
1871        b_strides,
1872        f,
1873        validated,
1874        exec,
1875    )
1876}
1877
1878// The same three raw-parts operands plus the closure, the `validated` marker and the
1879// token: the planned traversal for the whole operation, entered once, not per block.
1880#[allow(clippy::too_many_arguments)]
1881fn zip_map2_parts_into_validated<
1882    D: Copy + Send + Sync,
1883    A: Copy + Send + Sync,
1884    B: Copy + Send + Sync,
1885    OpA: ElementOp<A>,
1886    OpB: ElementOp<B>,
1887>(
1888    dst_ptr: *mut D,
1889    dst_dims: &[usize],
1890    dst_strides: &[isize],
1891    a_ptr: *const A,
1892    a_strides: &[isize],
1893    b_ptr: *const B,
1894    b_strides: &[isize],
1895    f: impl Fn(A, B) -> D + Sync,
1896    _validated: ValidatedDestinationLayout,
1897    exec: Exec<'_>,
1898) -> Result<()> {
1899    if sequential_contiguous_layout(
1900        KernelClass::Map,
1901        dst_dims,
1902        &[dst_strides, a_strides, b_strides],
1903        &exec,
1904    )?
1905    .is_some()
1906    {
1907        let len = total_len(dst_dims)?;
1908        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
1909        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
1910        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
1911        simd::dispatch_if_large(len, || {
1912            for i in 0..len {
1913                dst[i] = f(OpA::apply(sa[i]), OpB::apply(sb[i]));
1914            }
1915        });
1916        return Ok(());
1917    }
1918
1919    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
1920    let total = total_len(dst_dims)?;
1921    let (fused_dims, ordered_strides, plan) = pointwise_plan(
1922        dst_dims,
1923        &strides_list,
1924        Some(0),
1925        &[
1926            std::mem::size_of::<D>(),
1927            std::mem::size_of::<A>(),
1928            std::mem::size_of::<B>(),
1929        ],
1930        total,
1931    );
1932
1933    let total = total_len(&fused_dims)?;
1934    if is_parallel(KernelClass::Map, total, &exec) {
1935        use crate::threading::SendPtr;
1936        let dst_send = SendPtr(dst_ptr);
1937        let a_send = SendPtr(a_ptr as *mut A);
1938        let b_send = SendPtr(b_ptr as *mut B);
1939
1940        let costs = compute_costs(&ordered_strides);
1941        let initial_offsets: crate::inline::OperandVec<isize> =
1942            strides_list.iter().map(|_| 0isize).collect();
1943        return mapreduce_threaded(
1944            &fused_dims,
1945            &plan.block,
1946            &ordered_strides,
1947            &initial_offsets,
1948            &costs,
1949            KernelClass::Map,
1950            exec,
1951            &|dims, blocks, strides_list, offsets| {
1952                for_each_inner_block_with_offsets(
1953                    dims,
1954                    blocks,
1955                    strides_list,
1956                    offsets,
1957                    |offsets, len, strides| {
1958                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
1959                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
1960                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
1961                        unsafe {
1962                            inner_loop_map2::<D, A, B, OpA, OpB>(
1963                                dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1964                            )
1965                        };
1966                        Ok(())
1967                    },
1968                )
1969            },
1970        );
1971    }
1972
1973    let initial_offsets: crate::inline::OperandVec<isize> =
1974        ordered_strides.iter().map(|_| 0isize).collect();
1975    for_each_inner_block_preordered(
1976        &fused_dims,
1977        &plan.block,
1978        &ordered_strides,
1979        &initial_offsets,
1980        |offsets, len, strides| {
1981            let dp = unsafe { dst_ptr.offset(offsets[0]) };
1982            let ap = unsafe { a_ptr.offset(offsets[1]) };
1983            let bp = unsafe { b_ptr.offset(offsets[2]) };
1984            unsafe {
1985                inner_loop_map2::<D, A, B, OpA, OpB>(
1986                    dp, strides[0], ap, strides[1], bp, strides[2], len, &f,
1987                )
1988            };
1989            Ok(())
1990        },
1991    )
1992}
1993
1994// Raw parts for the whole multiplication: it takes the contiguous fast path when the three
1995// layouts agree, and otherwise walks the strided one. The operands here are the
1996// operation's, not a block's.
1997#[allow(clippy::too_many_arguments)]
1998fn mul_identity_into_raw<
1999    O: MulOutput<D>,
2000    D: Copy + Send + Sync + 'static,
2001    A: Copy + Send + Sync + Mul<B, Output = D> + 'static,
2002    B: Copy + Send + Sync + 'static,
2003>(
2004    dst_ptr: *mut O::Slot,
2005    dst_dims: &[usize],
2006    dst_strides: &[isize],
2007    a_ptr: *const A,
2008    a_strides: &[isize],
2009    b_ptr: *const B,
2010    b_strides: &[isize],
2011    _validated: ValidatedDestinationLayout,
2012    mut exec: Exec<'_>,
2013) -> Result<()> {
2014    debug_assert_eq!(dst_dims.len(), a_strides.len());
2015    debug_assert_eq!(dst_dims.len(), b_strides.len());
2016
2017    if sequential_contiguous_layout(
2018        KernelClass::Map,
2019        dst_dims,
2020        &[dst_strides, a_strides, b_strides],
2021        &exec,
2022    )?
2023    .is_some()
2024    {
2025        let len = total_len(dst_dims)?;
2026        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2027        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2028        if unsafe { O::try_contiguous(dst_ptr, len, sa, sb) } {
2029            return Ok(());
2030        }
2031        for i in 0..len {
2032            unsafe { O::write(dst_ptr.add(i), multiply_value(sa[i], sb[i])) };
2033        }
2034        return Ok(());
2035    }
2036
2037    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
2038    let total = total_len(dst_dims)?;
2039
2040    if try_contiguous_range_mul::<O, D, A, B>(
2041        dst_ptr,
2042        dst_dims,
2043        dst_strides,
2044        a_ptr,
2045        a_strides,
2046        b_ptr,
2047        b_strides,
2048        exec.reborrow(),
2049    ) {
2050        return Ok(());
2051    }
2052
2053    let (fused_dims, ordered_strides, plan) = pointwise_plan(
2054        dst_dims,
2055        &strides_list,
2056        Some(0),
2057        &[
2058            std::mem::size_of::<D>(),
2059            std::mem::size_of::<A>(),
2060            std::mem::size_of::<B>(),
2061        ],
2062        total,
2063    );
2064
2065    let total = total_len(&fused_dims)?;
2066    if is_parallel(KernelClass::Map, total, &exec) {
2067        use crate::threading::SendPtr;
2068        let dst_send = SendPtr(dst_ptr);
2069        let a_send = SendPtr(a_ptr as *mut A);
2070        let b_send = SendPtr(b_ptr as *mut B);
2071
2072        let costs = compute_costs(&ordered_strides);
2073        let initial_offsets: crate::inline::OperandVec<isize> =
2074            strides_list.iter().map(|_| 0isize).collect();
2075        return mapreduce_threaded(
2076            &fused_dims,
2077            &plan.block,
2078            &ordered_strides,
2079            &initial_offsets,
2080            &costs,
2081            KernelClass::Map,
2082            exec,
2083            &|dims, blocks, strides_list, offsets| {
2084                for_each_inner_block_with_offsets(
2085                    dims,
2086                    blocks,
2087                    strides_list,
2088                    offsets,
2089                    |offsets, len, strides| {
2090                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2091                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2092                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2093                        unsafe {
2094                            inner_loop_mul2::<O, D, A, B>(
2095                                dp, strides[0], ap, strides[1], bp, strides[2], len,
2096                            )
2097                        };
2098                        Ok(())
2099                    },
2100                )
2101            },
2102        );
2103    }
2104
2105    let initial_offsets: crate::inline::OperandVec<isize> =
2106        ordered_strides.iter().map(|_| 0isize).collect();
2107    for_each_inner_block_preordered(
2108        &fused_dims,
2109        &plan.block,
2110        &ordered_strides,
2111        &initial_offsets,
2112        |offsets, len, strides| {
2113            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2114            let ap = unsafe { a_ptr.offset(offsets[1]) };
2115            let bp = unsafe { b_ptr.offset(offsets[2]) };
2116            unsafe {
2117                inner_loop_mul2::<O, D, A, B>(dp, strides[0], ap, strides[1], bp, strides[2], len)
2118            };
2119            Ok(())
2120        },
2121    )
2122}
2123
2124/// Element-wise multiplication: `dest[i] = a[i] * b[i]`.
2125///
2126/// All views must have the same shape. Broadcast operands should be represented
2127/// as stride-0 views before calling this function.
2128///
2129/// # Errors
2130///
2131/// Returns [`StridedError::RankMismatch`] or
2132/// [`StridedError::ShapeMismatch`] when `dest` and either operand have
2133/// different ranks or extents, and
2134/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2135/// cannot be proven representable and injective: two logical indices name
2136/// one offset, or the element count or address span is too large to
2137/// represent. Nothing is written unless validation succeeds.
2138pub fn mul_into<
2139    D: Copy + Send + Sync + 'static,
2140    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2141    B: Copy + Send + Sync + 'static,
2142    OpA: ElementOp<A>,
2143    OpB: ElementOp<B>,
2144>(
2145    dest: &mut StridedViewMut<D>,
2146    a: &StridedView<A, OpA>,
2147    b: &StridedView<B, OpB>,
2148    exec: Exec<'_>,
2149) -> Result<()> {
2150    ensure_same_shape(dest.dims(), a.dims())?;
2151    ensure_same_shape(dest.dims(), b.dims())?;
2152    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2153
2154    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2155        return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2156            dest.as_mut_ptr(),
2157            dest.dims(),
2158            dest.strides(),
2159            a.ptr(),
2160            a.strides(),
2161            b.ptr(),
2162            b.strides(),
2163            validated,
2164            exec,
2165        );
2166    }
2167
2168    zip_map2_into_validated(dest, a, b, multiply_value, validated, exec)
2169}
2170
2171/// Multiply two views into a fully overwritten uninitialized output.
2172///
2173/// Shape, destination-injectivity, and reachable-byte overlap validation
2174/// completes before the first write. Safe Rust borrows already prevent
2175/// input/output aliasing; the explicit overlap check preserves the contract for
2176/// views produced through unsafe constructors.
2177///
2178/// `Ok(())` means every logical destination element is initialized. An error
2179/// occurs before writes. A panic during replay may leave a partially initialized
2180/// destination, which remains safe to drop as `MaybeUninit<D>`.
2181///
2182/// # Errors
2183///
2184/// Returns [`StridedError::ShapeMismatch`] for unequal shapes,
2185/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
2186/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
2187/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
2188/// representable.
2189pub fn mul_into_uninit<
2190    D: Copy + Send + Sync + 'static,
2191    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2192    B: Copy + Send + Sync + 'static,
2193    OpA: ElementOp<A>,
2194    OpB: ElementOp<B>,
2195>(
2196    dest: &mut StridedViewMut<MaybeUninit<D>>,
2197    a: &StridedView<A, OpA>,
2198    b: &StridedView<B, OpB>,
2199    exec: Exec<'_>,
2200) -> Result<()> {
2201    ensure_same_shape(dest.dims(), a.dims())?;
2202    ensure_same_shape(dest.dims(), b.dims())?;
2203    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2204    validate_typed_no_overlap(dest, a, 0)?;
2205    validate_typed_no_overlap(dest, b, 1)?;
2206
2207    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2208        return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2209            dest.as_mut_ptr(),
2210            dest.dims(),
2211            dest.strides(),
2212            a.ptr(),
2213            a.strides(),
2214            b.ptr(),
2215            b.strides(),
2216            validated,
2217            exec,
2218        );
2219    }
2220
2221    zip_map2_into_validated(
2222        dest,
2223        a,
2224        b,
2225        |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2226        validated,
2227        exec,
2228    )
2229}
2230
2231fn broadcast_strides_for_axes(
2232    source_dims: &[usize],
2233    source_strides: &[isize],
2234    target_dims: &[usize],
2235    axes: &[usize],
2236) -> Result<AxisVec<isize>> {
2237    if source_dims.len() != axes.len() {
2238        return Err(StridedError::RankMismatch(source_dims.len(), axes.len()));
2239    }
2240    debug_assert_eq!(source_dims.len(), source_strides.len());
2241
2242    let mut seen = AxisVec::<bool>::new();
2243    seen.resize(target_dims.len(), false);
2244    let mut strides = AxisVec::<isize>::new();
2245    strides.resize(target_dims.len(), 0);
2246    for (src_axis, &dst_axis) in axes.iter().enumerate() {
2247        if dst_axis >= target_dims.len() {
2248            return Err(StridedError::InvalidAxis {
2249                axis: dst_axis,
2250                rank: target_dims.len(),
2251            });
2252        }
2253        if seen[dst_axis] {
2254            return Err(StridedError::InvalidAxis {
2255                axis: dst_axis,
2256                rank: target_dims.len(),
2257            });
2258        }
2259        seen[dst_axis] = true;
2260
2261        let source_dim = source_dims[src_axis];
2262        let target_dim = target_dims[dst_axis];
2263        if source_dim != target_dim && source_dim != 1 {
2264            return Err(StridedError::ShapeMismatch(
2265                source_dims.to_vec(),
2266                target_dims.to_vec(),
2267            ));
2268        }
2269        if source_dim == target_dim {
2270            strides[dst_axis] = source_strides[src_axis];
2271        }
2272    }
2273
2274    Ok(strides)
2275}
2276
2277/// A view of `view`'s data with stride-0 broadcast axes.
2278///
2279/// The metadata is computed for this call only; [`StridedView::new_unchecked`]
2280/// copies it into owned metadata (inline up to [`INLINE_RANK`](crate::INLINE_RANK)
2281/// axes, boxed above, so a rank-9 view allocates), and the view still borrows
2282/// only the element data.
2283fn broadcast_view_with_strides<'a, T, Op: ElementOp<T>>(
2284    view: &StridedView<'a, T, Op>,
2285    target_dims: &[usize],
2286    strides: &[isize],
2287) -> StridedView<'a, T, Op> {
2288    // SAFETY: `broadcast_strides_for_axes` (:2231) keeps a source stride only on
2289    // axes where `source_dims[src_axis] == target_dims[dst_axis]`, sets every other
2290    // target axis to stride 0, rejects repeated and out-of-range target axes, and
2291    // requires `source_dims.len() == axes.len()`, so this view reaches a subset of
2292    // the offsets `view` reaches; `StridedView` construction (`new`,
2293    // strided-view/src/view.rs:509 via `validate_bounds`, :187) already proved those
2294    // in bounds, and `target_dims`/`strides` come from the caller's `dest.dims()`
2295    // and that validation. `new_unchecked` copies the metadata into owned
2296    // storage, not inline: `ViewMeta::from_slices` (strided-view/src/view.rs:129)
2297    // keeps it inline up to `INLINE_RANK` = 8 and boxes it above (strided-view/src/view.rs:141), which
2298    // is why a rank-9 call allocates here.
2299    unsafe { StridedView::new_unchecked(view.data(), target_dims, strides, view.offset()) }
2300}
2301
2302/// Broadcasted element-wise multiplication: `dest[i] = a[i] * b[i]`.
2303///
2304/// `a_axes` and `b_axes` map each source axis to an axis of `dest`. Output axes
2305/// not referenced by a source operand are treated as stride-0 broadcast axes.
2306///
2307/// # Errors
2308///
2309/// Returns [`StridedError::RankMismatch`] when an axis list does not have one
2310/// entry per source axis, [`StridedError::InvalidAxis`] when an axis lies
2311/// outside `dest` or is mapped twice, [`StridedError::ShapeMismatch`] when a
2312/// source extent is neither the destination extent nor 1, and
2313/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2314/// cannot be proven representable and injective: two logical indices name
2315/// one offset, or the element count or address span is too large to
2316/// represent. Nothing is written unless validation succeeds.
2317pub fn broadcast_mul_into<
2318    D: Copy + Send + Sync + 'static,
2319    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2320    B: Copy + Send + Sync + 'static,
2321    OpA: ElementOp<A>,
2322    OpB: ElementOp<B>,
2323>(
2324    dest: &mut StridedViewMut<D>,
2325    a: &StridedView<A, OpA>,
2326    a_axes: &[usize],
2327    b: &StridedView<B, OpB>,
2328    b_axes: &[usize],
2329    exec: Exec<'_>,
2330) -> Result<()> {
2331    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2332    let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2333    let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2334
2335    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2336        return mul_identity_into_raw::<InitializedOutput, D, A, B>(
2337            dest.as_mut_ptr(),
2338            dest.dims(),
2339            dest.strides(),
2340            a.ptr(),
2341            &a_strides,
2342            b.ptr(),
2343            &b_strides,
2344            validated,
2345            exec,
2346        );
2347    }
2348
2349    let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2350    let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2351    zip_map2_into_validated(dest, &a, &b, multiply_value, validated, exec)
2352}
2353
2354/// Broadcast and multiply into a fully overwritten uninitialized output.
2355///
2356/// Axis mappings, shapes, destination injectivity, and reachable-byte overlap
2357/// are validated before the first write. Safe Rust borrows already prevent
2358/// input/output aliasing; the explicit overlap check preserves the contract for
2359/// views produced through unsafe constructors.
2360///
2361/// `Ok(())` means every logical destination element is initialized. An error
2362/// occurs before writes. A panic during replay may leave a partially initialized
2363/// destination, which remains safe to drop as `MaybeUninit<D>`.
2364///
2365/// # Errors
2366///
2367/// Returns a typed rank, axis, or shape error for an invalid broadcast mapping,
2368/// [`StridedError::NonInjectiveOutputLayout`] for an overlapping output layout,
2369/// [`StridedError::OverlappingInputOutput`] for aliased storage, or
2370/// [`StridedError::OffsetOverflow`] when a reachable byte range is not
2371/// representable.
2372pub fn broadcast_mul_into_uninit<
2373    D: Copy + Send + Sync + 'static,
2374    A: Copy + Mul<B, Output = D> + Send + Sync + 'static,
2375    B: Copy + Send + Sync + 'static,
2376    OpA: ElementOp<A>,
2377    OpB: ElementOp<B>,
2378>(
2379    dest: &mut StridedViewMut<MaybeUninit<D>>,
2380    a: &StridedView<A, OpA>,
2381    a_axes: &[usize],
2382    b: &StridedView<B, OpB>,
2383    b_axes: &[usize],
2384    exec: Exec<'_>,
2385) -> Result<()> {
2386    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2387    let a_strides = broadcast_strides_for_axes(a.dims(), a.strides(), dest.dims(), a_axes)?;
2388    let b_strides = broadcast_strides_for_axes(b.dims(), b.strides(), dest.dims(), b_axes)?;
2389    validate_typed_no_overlap(dest, a, 0)?;
2390    validate_typed_no_overlap(dest, b, 1)?;
2391
2392    if OpA::IS_IDENTITY && OpB::IS_IDENTITY {
2393        return mul_identity_into_raw::<UninitializedOutput, D, A, B>(
2394            dest.as_mut_ptr(),
2395            dest.dims(),
2396            dest.strides(),
2397            a.ptr(),
2398            &a_strides,
2399            b.ptr(),
2400            &b_strides,
2401            validated,
2402            exec,
2403        );
2404    }
2405
2406    let a = broadcast_view_with_strides(a, dest.dims(), &a_strides);
2407    let b = broadcast_view_with_strides(b, dest.dims(), &b_strides);
2408    zip_map2_into_validated(
2409        dest,
2410        &a,
2411        &b,
2412        |lhs, rhs| MaybeUninit::new(multiply_value(lhs, rhs)),
2413        validated,
2414        exec,
2415    )
2416}
2417
2418/// Ternary element-wise operation: `dest[i] = f(a[i], b[i], c[i])`.
2419///
2420/// # Errors
2421///
2422/// Returns [`StridedError::RankMismatch`] or
2423/// [`StridedError::ShapeMismatch`] when `dest` and any of the three source
2424/// views have different ranks or extents, and
2425/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2426/// cannot be proven representable and injective: two logical indices name
2427/// one offset, or the element count or address span is too large to
2428/// represent. Nothing is written unless validation succeeds.
2429pub fn zip_map3_into<
2430    D: Copy + Send + Sync,
2431    A: Copy + Send + Sync,
2432    B: Copy + Send + Sync,
2433    C: Copy + Send + Sync,
2434    OpA: ElementOp<A>,
2435    OpB: ElementOp<B>,
2436    OpC: ElementOp<C>,
2437>(
2438    dest: &mut StridedViewMut<D>,
2439    a: &StridedView<A, OpA>,
2440    b: &StridedView<B, OpB>,
2441    c: &StridedView<C, OpC>,
2442    f: impl Fn(A, B, C) -> D + Sync,
2443    exec: Exec<'_>,
2444) -> Result<()> {
2445    ensure_same_shape(dest.dims(), a.dims())?;
2446    ensure_same_shape(dest.dims(), b.dims())?;
2447    ensure_same_shape(dest.dims(), c.dims())?;
2448    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2449    zip_map3_into_validated(dest, a, b, c, f, validated, exec)
2450}
2451
2452pub(crate) fn zip_map3_into_validated<
2453    D: Copy + Send + Sync,
2454    A: Copy + Send + Sync,
2455    B: Copy + Send + Sync,
2456    C: Copy + Send + Sync,
2457    OpA: ElementOp<A>,
2458    OpB: ElementOp<B>,
2459    OpC: ElementOp<C>,
2460>(
2461    dest: &mut StridedViewMut<D>,
2462    a: &StridedView<A, OpA>,
2463    b: &StridedView<B, OpB>,
2464    c: &StridedView<C, OpC>,
2465    f: impl Fn(A, B, C) -> D + Sync,
2466    _validated: ValidatedDestinationLayout,
2467    exec: Exec<'_>,
2468) -> Result<()> {
2469    ensure_same_shape(dest.dims(), a.dims())?;
2470    ensure_same_shape(dest.dims(), b.dims())?;
2471    ensure_same_shape(dest.dims(), c.dims())?;
2472    let dst_ptr = dest.as_mut_ptr();
2473    let a_ptr = a.ptr();
2474    let b_ptr = b.ptr();
2475    let c_ptr = c.ptr();
2476
2477    let dst_dims = dest.dims();
2478    let dst_strides = dest.strides();
2479
2480    if sequential_contiguous_layout(
2481        KernelClass::Map,
2482        dst_dims,
2483        &[dst_strides, a.strides(), b.strides(), c.strides()],
2484        &exec,
2485    )?
2486    .is_some()
2487    {
2488        let len = total_len(dst_dims)?;
2489        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2490        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2491        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2492        let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2493        simd::dispatch_if_large(len, || {
2494            for (((d, &a), &b), &c) in dst.iter_mut().zip(sa).zip(sb).zip(sc) {
2495                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c));
2496            }
2497        });
2498        return Ok(());
2499    }
2500
2501    let strides_list: [&[isize]; 4] = [dst_strides, a.strides(), b.strides(), c.strides()];
2502    let total = total_len(dst_dims)?;
2503    let (fused_dims, ordered_strides, plan) = pointwise_plan(
2504        dst_dims,
2505        &strides_list,
2506        Some(0),
2507        &[
2508            std::mem::size_of::<D>(),
2509            std::mem::size_of::<A>(),
2510            std::mem::size_of::<B>(),
2511            std::mem::size_of::<C>(),
2512        ],
2513        total,
2514    );
2515
2516    let total = total_len(&fused_dims)?;
2517    if is_parallel(KernelClass::Map, total, &exec) {
2518        use crate::threading::SendPtr;
2519        let dst_send = SendPtr(dst_ptr);
2520        let a_send = SendPtr(a_ptr as *mut A);
2521        let b_send = SendPtr(b_ptr as *mut B);
2522        let c_send = SendPtr(c_ptr as *mut C);
2523
2524        let costs = compute_costs(&ordered_strides);
2525        let initial_offsets: crate::inline::OperandVec<isize> =
2526            strides_list.iter().map(|_| 0isize).collect();
2527        return mapreduce_threaded(
2528            &fused_dims,
2529            &plan.block,
2530            &ordered_strides,
2531            &initial_offsets,
2532            &costs,
2533            KernelClass::Map,
2534            exec,
2535            &|dims, blocks, strides_list, offsets| {
2536                for_each_inner_block_with_offsets(
2537                    dims,
2538                    blocks,
2539                    strides_list,
2540                    offsets,
2541                    |offsets, len, strides| {
2542                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2543                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2544                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2545                        let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2546                        unsafe {
2547                            inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2548                                dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3],
2549                                len, &f,
2550                            )
2551                        };
2552                        Ok(())
2553                    },
2554                )
2555            },
2556        );
2557    }
2558
2559    let initial_offsets: crate::inline::OperandVec<isize> =
2560        ordered_strides.iter().map(|_| 0isize).collect();
2561    for_each_inner_block_preordered(
2562        &fused_dims,
2563        &plan.block,
2564        &ordered_strides,
2565        &initial_offsets,
2566        |offsets, len, strides| {
2567            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2568            let ap = unsafe { a_ptr.offset(offsets[1]) };
2569            let bp = unsafe { b_ptr.offset(offsets[2]) };
2570            let cp = unsafe { c_ptr.offset(offsets[3]) };
2571            unsafe {
2572                inner_loop_map3::<D, A, B, C, OpA, OpB, OpC>(
2573                    dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], len, &f,
2574                )
2575            };
2576            Ok(())
2577        },
2578    )
2579}
2580
2581/// Quaternary element-wise operation: `dest[i] = f(a[i], b[i], c[i], e[i])`.
2582///
2583/// # Errors
2584///
2585/// Returns [`StridedError::RankMismatch`] or
2586/// [`StridedError::ShapeMismatch`] when `dest` and any of the four source
2587/// views have different ranks or extents, and
2588/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
2589/// cannot be proven representable and injective: two logical indices name
2590/// one offset, or the element count or address span is too large to
2591/// represent. Nothing is written unless validation succeeds.
2592pub fn zip_map4_into<
2593    D: Copy + Send + Sync,
2594    A: Copy + Send + Sync,
2595    B: Copy + Send + Sync,
2596    C: Copy + Send + Sync,
2597    E: Copy + Send + Sync,
2598    OpA: ElementOp<A>,
2599    OpB: ElementOp<B>,
2600    OpC: ElementOp<C>,
2601    OpE: ElementOp<E>,
2602>(
2603    dest: &mut StridedViewMut<D>,
2604    a: &StridedView<A, OpA>,
2605    b: &StridedView<B, OpB>,
2606    c: &StridedView<C, OpC>,
2607    e: &StridedView<E, OpE>,
2608    f: impl Fn(A, B, C, E) -> D + Sync,
2609    exec: Exec<'_>,
2610) -> Result<()> {
2611    ensure_same_shape(dest.dims(), a.dims())?;
2612    ensure_same_shape(dest.dims(), b.dims())?;
2613    ensure_same_shape(dest.dims(), c.dims())?;
2614    ensure_same_shape(dest.dims(), e.dims())?;
2615    let validated = validate_destination_layout(dest.dims(), dest.strides())?;
2616    zip_map4_into_validated(dest, a, b, c, e, f, validated, exec)
2617}
2618
2619// Four operand views plus the destination, the closure, the `validated` marker and
2620// the token: the entry's shape.
2621#[allow(clippy::too_many_arguments)]
2622pub(crate) fn zip_map4_into_validated<
2623    D: Copy + Send + Sync,
2624    A: Copy + Send + Sync,
2625    B: Copy + Send + Sync,
2626    C: Copy + Send + Sync,
2627    E: Copy + Send + Sync,
2628    OpA: ElementOp<A>,
2629    OpB: ElementOp<B>,
2630    OpC: ElementOp<C>,
2631    OpE: ElementOp<E>,
2632>(
2633    dest: &mut StridedViewMut<D>,
2634    a: &StridedView<A, OpA>,
2635    b: &StridedView<B, OpB>,
2636    c: &StridedView<C, OpC>,
2637    e: &StridedView<E, OpE>,
2638    f: impl Fn(A, B, C, E) -> D + Sync,
2639    _validated: ValidatedDestinationLayout,
2640    exec: Exec<'_>,
2641) -> Result<()> {
2642    ensure_same_shape(dest.dims(), a.dims())?;
2643    ensure_same_shape(dest.dims(), b.dims())?;
2644    ensure_same_shape(dest.dims(), c.dims())?;
2645    ensure_same_shape(dest.dims(), e.dims())?;
2646    let dst_ptr = dest.as_mut_ptr();
2647    let a_ptr = a.ptr();
2648    let b_ptr = b.ptr();
2649    let c_ptr = c.ptr();
2650    let e_ptr = e.ptr();
2651
2652    let dst_dims = dest.dims();
2653    let dst_strides = dest.strides();
2654
2655    if sequential_contiguous_layout(
2656        KernelClass::Map,
2657        dst_dims,
2658        &[
2659            dst_strides,
2660            a.strides(),
2661            b.strides(),
2662            c.strides(),
2663            e.strides(),
2664        ],
2665        &exec,
2666    )?
2667    .is_some()
2668    {
2669        let len = total_len(dst_dims)?;
2670        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
2671        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
2672        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
2673        let sc = unsafe { std::slice::from_raw_parts(c_ptr, len) };
2674        let se = unsafe { std::slice::from_raw_parts(e_ptr, len) };
2675        // As in `inner_loop_map4`: zipped iterators keep the dispatched closure
2676        // small enough to inline (tensor4all/strided-rs#340).
2677        simd::dispatch_if_large(len, || {
2678            for ((((d, &a), &b), &c), &e) in dst.iter_mut().zip(sa).zip(sb).zip(sc).zip(se) {
2679                *d = f(OpA::apply(a), OpB::apply(b), OpC::apply(c), OpE::apply(e));
2680            }
2681        });
2682        return Ok(());
2683    }
2684
2685    let strides_list: [&[isize]; 5] = [
2686        dst_strides,
2687        a.strides(),
2688        b.strides(),
2689        c.strides(),
2690        e.strides(),
2691    ];
2692    let total = total_len(dst_dims)?;
2693    let (fused_dims, ordered_strides, plan) = pointwise_plan(
2694        dst_dims,
2695        &strides_list,
2696        Some(0),
2697        &[
2698            std::mem::size_of::<D>(),
2699            std::mem::size_of::<A>(),
2700            std::mem::size_of::<B>(),
2701            std::mem::size_of::<C>(),
2702            std::mem::size_of::<E>(),
2703        ],
2704        total,
2705    );
2706
2707    let total = total_len(&fused_dims)?;
2708    if is_parallel(KernelClass::Map, total, &exec) {
2709        use crate::threading::SendPtr;
2710        let dst_send = SendPtr(dst_ptr);
2711        let a_send = SendPtr(a_ptr as *mut A);
2712        let b_send = SendPtr(b_ptr as *mut B);
2713        let c_send = SendPtr(c_ptr as *mut C);
2714        let e_send = SendPtr(e_ptr as *mut E);
2715
2716        let costs = compute_costs(&ordered_strides);
2717        let initial_offsets: crate::inline::OperandVec<isize> =
2718            strides_list.iter().map(|_| 0isize).collect();
2719        return mapreduce_threaded(
2720            &fused_dims,
2721            &plan.block,
2722            &ordered_strides,
2723            &initial_offsets,
2724            &costs,
2725            KernelClass::Map,
2726            exec,
2727            &|dims, blocks, strides_list, offsets| {
2728                for_each_inner_block_with_offsets(
2729                    dims,
2730                    blocks,
2731                    strides_list,
2732                    offsets,
2733                    |offsets, len, strides| {
2734                        let dp = unsafe { dst_send.as_ptr().offset(offsets[0]) };
2735                        let ap = unsafe { a_send.as_const().offset(offsets[1]) };
2736                        let bp = unsafe { b_send.as_const().offset(offsets[2]) };
2737                        let cp = unsafe { c_send.as_const().offset(offsets[3]) };
2738                        let ep = unsafe { e_send.as_const().offset(offsets[4]) };
2739                        unsafe {
2740                            inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2741                                dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep,
2742                                strides[4], len, &f,
2743                            )
2744                        };
2745                        Ok(())
2746                    },
2747                )
2748            },
2749        );
2750    }
2751
2752    let initial_offsets: crate::inline::OperandVec<isize> =
2753        ordered_strides.iter().map(|_| 0isize).collect();
2754    for_each_inner_block_preordered(
2755        &fused_dims,
2756        &plan.block,
2757        &ordered_strides,
2758        &initial_offsets,
2759        |offsets, len, strides| {
2760            let dp = unsafe { dst_ptr.offset(offsets[0]) };
2761            let ap = unsafe { a_ptr.offset(offsets[1]) };
2762            let bp = unsafe { b_ptr.offset(offsets[2]) };
2763            let cp = unsafe { c_ptr.offset(offsets[3]) };
2764            let ep = unsafe { e_ptr.offset(offsets[4]) };
2765            unsafe {
2766                inner_loop_map4::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
2767                    dp, strides[0], ap, strides[1], bp, strides[2], cp, strides[3], ep, strides[4],
2768                    len, &f,
2769                )
2770            };
2771            Ok(())
2772        },
2773    )
2774}
2775
2776#[cfg(test)]
2777#[path = "map_view/tests/scalar_branch_tests.rs"]
2778mod scalar_branch_tests;
2779
2780#[cfg(test)]
2781#[path = "map_view/tests/tests.rs"]
2782mod tests;