Skip to main content

strided_basic/
ops_view.rs

1//! High-level operations on dynamic-rank strided views.
2
3use crate::kernel::{
4    build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
5    same_contiguous_layout, sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
6};
7use crate::map_view::{copy_map_into, map_into, zip_map2_into};
8use crate::reduce_view::reduce;
9use crate::simd;
10use crate::view::{StridedView, StridedViewMut};
11use crate::{Exec, Result, StridedError};
12use num_traits::{One, Zero};
13use std::mem::MaybeUninit;
14use std::ops::{Add, Mul};
15use strided_view::{ElementOp, ElementOpApply};
16
17use crate::cost::KernelClass;
18use crate::fuse::compute_costs;
19use crate::threading::{
20    for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded, SendPtr,
21};
22// ============================================================================
23// Stride-specialized inner loop helpers for ops_view
24//
25// When all inner strides are 1 (contiguous in the innermost dimension),
26// we use slice-based iteration so LLVM can auto-vectorize effectively.
27// This mirrors the inner_loop_map* helpers in map_view.rs.
28// ============================================================================
29
30// INVARIANT: validated layouts keep all dereferences in bounds. Wrapping
31// advances permit an unused final cursor outside a reversed/gapped allocation.
32/// Inner loop for `add_acc`: `dst[i] += Op::apply(src[i])`.
33#[inline(always)]
34unsafe fn inner_loop_add<D: Copy + Add<S, Output = D>, S: Copy, Op: ElementOp<S>>(
35    dp: *mut D,
36    ds: isize,
37    sp: *const S,
38    ss: isize,
39    len: usize,
40) {
41    if ds == 1 && ss == 1 {
42        let dst = std::slice::from_raw_parts_mut(dp, len);
43        let src = std::slice::from_raw_parts(sp, len);
44        simd::dispatch_if_large(len, || {
45            for i in 0..len {
46                dst[i] = dst[i] + Op::apply(src[i]);
47            }
48        });
49    } else {
50        let mut dp = dp;
51        let mut sp = sp;
52        for _ in 0..len {
53            *dp = *dp + Op::apply(*sp);
54            dp = dp.wrapping_offset(ds);
55            sp = sp.wrapping_offset(ss);
56        }
57    }
58}
59
60/// Inner loop for `mul_acc`: `dst[i] *= Op::apply(src[i])`.
61#[inline(always)]
62unsafe fn inner_loop_mul<D: Copy + Mul<S, Output = D>, S: Copy, Op: ElementOp<S>>(
63    dp: *mut D,
64    ds: isize,
65    sp: *const S,
66    ss: isize,
67    len: usize,
68) {
69    if ds == 1 && ss == 1 {
70        let dst = std::slice::from_raw_parts_mut(dp, len);
71        let src = std::slice::from_raw_parts(sp, len);
72        simd::dispatch_if_large(len, || {
73            for i in 0..len {
74                dst[i] = dst[i] * Op::apply(src[i]);
75            }
76        });
77    } else {
78        let mut dp = dp;
79        let mut sp = sp;
80        for _ in 0..len {
81            *dp = *dp * Op::apply(*sp);
82            dp = dp.wrapping_offset(ds);
83            sp = sp.wrapping_offset(ss);
84        }
85    }
86}
87
88/// Inner loop for `axpy_acc`: `dst[i] = alpha * Op::apply(src[i]) + dst[i]`.
89#[inline(always)]
90unsafe fn inner_loop_axpy<
91    D: Copy + Add<D, Output = D>,
92    S: Copy,
93    A: Copy + Mul<S, Output = D>,
94    Op: ElementOp<S>,
95>(
96    dp: *mut D,
97    ds: isize,
98    sp: *const S,
99    ss: isize,
100    len: usize,
101    alpha: A,
102) {
103    if ds == 1 && ss == 1 {
104        let dst = std::slice::from_raw_parts_mut(dp, len);
105        let src = std::slice::from_raw_parts(sp, len);
106        simd::dispatch_if_large(len, || {
107            for i in 0..len {
108                dst[i] = alpha * Op::apply(src[i]) + dst[i];
109            }
110        });
111    } else {
112        let mut dp = dp;
113        let mut sp = sp;
114        for _ in 0..len {
115            *dp = alpha * Op::apply(*sp) + *dp;
116            dp = dp.wrapping_offset(ds);
117            sp = sp.wrapping_offset(ss);
118        }
119    }
120}
121
122/// Inner loop for `fma_acc`: `dst[i] += OpA::apply(a[i]) * OpB::apply(b[i])`.
123#[inline(always)]
124unsafe fn inner_loop_fma<
125    D: Copy + Add<D, Output = D>,
126    A: Copy + Mul<B, Output = D>,
127    B: Copy,
128    OpA: ElementOp<A>,
129    OpB: ElementOp<B>,
130>(
131    dp: *mut D,
132    ds: isize,
133    ap: *const A,
134    a_s: isize,
135    bp: *const B,
136    b_s: isize,
137    len: usize,
138) {
139    if ds == 1 && a_s == 1 && b_s == 1 {
140        let dst = std::slice::from_raw_parts_mut(dp, len);
141        let sa = std::slice::from_raw_parts(ap, len);
142        let sb = std::slice::from_raw_parts(bp, len);
143        simd::dispatch_if_large(len, || {
144            for i in 0..len {
145                dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
146            }
147        });
148    } else {
149        let mut dp = dp;
150        let mut ap = ap;
151        let mut bp = bp;
152        for _ in 0..len {
153            *dp = *dp + OpA::apply(*ap) * OpB::apply(*bp);
154            dp = dp.wrapping_offset(ds);
155            ap = ap.wrapping_offset(a_s);
156            bp = bp.wrapping_offset(b_s);
157        }
158    }
159}
160
161/// Inner loop for dot: `acc += OpA::apply(a[i]) * OpB::apply(b[i])`.
162#[inline(always)]
163unsafe fn inner_loop_dot<
164    A: Copy + Mul<B, Output = R>,
165    B: Copy,
166    R: Copy + Add<R, Output = R>,
167    OpA: ElementOp<A>,
168    OpB: ElementOp<B>,
169>(
170    ap: *const A,
171    a_s: isize,
172    bp: *const B,
173    b_s: isize,
174    len: usize,
175    mut acc: R,
176) -> R {
177    if a_s == 1 && b_s == 1 {
178        let sa = std::slice::from_raw_parts(ap, len);
179        let sb = std::slice::from_raw_parts(bp, len);
180        simd::dispatch_if_large(len, || {
181            for i in 0..len {
182                acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
183            }
184        });
185    } else {
186        let mut ap = ap;
187        let mut bp = bp;
188        for _ in 0..len {
189            acc = acc + OpA::apply(*ap) * OpB::apply(*bp);
190            ap = ap.wrapping_offset(a_s);
191            bp = bp.wrapping_offset(b_s);
192        }
193    }
194    acc
195}
196
197/// Copy elements from source to destination: `dest[i] = src[i]`.
198///
199/// # Errors
200///
201/// Returns [`StridedError::RankMismatch`] or
202/// [`StridedError::ShapeMismatch`] when the views describe different
203/// extents, and [`StridedError::OffsetOverflow`] when the element count of
204/// the shape does not fit in `usize`. The generic element path also returns
205/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
206/// cannot be proven representable and injective: two logical indices name
207/// one offset, or the address span is too large to represent. Nothing is
208/// written unless validation succeeds.
209pub fn copy_into<T: Copy + Send + Sync, Op: ElementOp<T>>(
210    dest: &mut StridedViewMut<T>,
211    src: &StridedView<T, Op>,
212    exec: Exec<'_>,
213) -> Result<()> {
214    ensure_same_shape(dest.dims(), src.dims())?;
215
216    let dst_ptr = dest.as_mut_ptr();
217    let src_ptr = src.ptr();
218    let dst_dims = dest.dims();
219    let dst_strides = dest.strides();
220    let src_strides = src.strides();
221
222    if sequential_contiguous_layout(
223        KernelClass::Copy,
224        dst_dims,
225        &[dst_strides, src_strides],
226        &exec,
227    )?
228    .is_some()
229    {
230        let len = total_len(dst_dims)?;
231        if Op::IS_IDENTITY {
232            debug_assert!(
233                {
234                    let nbytes = len
235                        .checked_mul(std::mem::size_of::<T>())
236                        .expect("copy size must not overflow");
237                    let dst_start = dst_ptr as usize;
238                    let src_start = src_ptr as usize;
239                    let dst_end = dst_start.saturating_add(nbytes);
240                    let src_end = src_start.saturating_add(nbytes);
241                    dst_end <= src_start || src_end <= dst_start
242                },
243                "overlapping src/dest is not supported"
244            );
245            unsafe { std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len) };
246        } else {
247            let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
248            let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
249            simd::dispatch_if_large(len, || {
250                for i in 0..len {
251                    dst[i] = Op::apply(src[i]);
252                }
253            });
254        }
255        return Ok(());
256    }
257
258    copy_map_into(dest, src, |x| x, exec)
259}
260
261/// Copy initialized values into a potentially uninitialized destination.
262///
263/// On success every logical destination element is initialized; holes are
264/// untouched. Accepts identity views; use `map_into` for element operations
265/// such as conjugation. Equal contiguous f32/f64 layouts and every other type
266/// keep the element map; a non-contiguous f32/f64 copy uses the permutation
267/// engine, split within the budget of `exec`.
268///
269/// # Errors
270/// Returns a shape/rank error for mismatched views, `NonInjectiveOutputLayout`
271/// for overlapping destination elements, or `OffsetOverflow` for size overflow.
272///
273/// # Examples
274/// ```
275/// use std::mem::MaybeUninit;
276/// use strided_basic::{copy_into_uninit, Exec, StridedView, StridedViewMut};
277/// let src = [1.0_f64, 2.0];
278/// let mut dst = [MaybeUninit::uninit(); 2];
279/// copy_into_uninit(
280///     &mut StridedViewMut::new(&mut dst, &[2], &[1], 0).unwrap(),
281///     &StridedView::new(&src, &[2], &[1], 0).unwrap(),
282///     Exec::serial(),
283/// ).unwrap();
284/// // SAFETY: the successful copy initialized both destination elements.
285/// assert_eq!(unsafe { dst[1].assume_init() }, 2.0);
286/// ```
287pub fn copy_into_uninit<T: Copy + Send + Sync + 'static>(
288    dest: &mut StridedViewMut<MaybeUninit<T>>,
289    src: &StridedView<T>,
290    exec: Exec<'_>,
291) -> Result<()> {
292    ensure_same_shape(dest.dims(), src.dims())?;
293    if !crate::layout_check::is_injective_layout(dest.dims(), dest.strides()) {
294        return Err(StridedError::NonInjectiveOutputLayout);
295    }
296    if src.is_empty() {
297        return Ok(());
298    }
299    let _len = src
300        .dims()
301        .iter()
302        .try_fold(1usize, |n, &dim| n.checked_mul(dim))
303        .ok_or(StridedError::OffsetOverflow)?;
304    // The permutation engine's 4/8-byte paths reinterpret storage as native
305    // floats. Restrict that path to those exact types: arbitrary Copy types can
306    // contain uninitialized padding or have weaker alignment (e.g. Complex32).
307    let native_float = std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>()
308        || std::any::TypeId::of::<T>() == std::any::TypeId::of::<f64>();
309    // Equal contiguous layouts go to the element map, whose contiguous path
310    // parallelizes a large copy; the permutation engine has only a serial
311    // contiguous shortcut. Non-contiguous native-float copies use the
312    // permutation engine, serial or parallel.
313    if !native_float
314        || same_contiguous_layout(dest.dims(), &[dest.strides(), src.strides()]).is_some()
315    {
316        return copy_map_into(dest, src, MaybeUninit::new, exec);
317    }
318    // INVARIANT: the permutation engine selects its float paths by the
319    // source element type and only stores into the destination, so an
320    // uninitialized destination with an initialized source is valid.
321    crate::threading::copy_permuted_uninit(dest, src, exec)
322}
323
324/// Copy elements from `src` to `dst`, optimized for col-major destination.
325///
326/// Delegates to the centralized permutation-copy dispatch for the actual work.
327///
328/// # Errors
329///
330/// The rejection set of that dispatch: [`StridedError::RankMismatch`] or
331/// [`StridedError::ShapeMismatch`] when the views describe different
332/// extents, [`StridedError::OffsetOverflow`] when the layout cannot be
333/// expressed in `isize` stride arithmetic or a fused extent overflows, and
334/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
335/// cannot be proven representable and injective. Nothing is written unless
336/// validation succeeds.
337pub fn copy_into_col_major<T: Copy + Send + Sync + 'static>(
338    dst: &mut StridedViewMut<T>,
339    src: &StridedView<T>,
340) -> Result<()> {
341    strided_perm::copy_into_col_major(dst, src, Exec::serial())
342}
343
344/// Accumulating element-wise addition: `dest[i] += src[i]`.
345///
346/// Source may have a different element type from destination.
347///
348/// # Errors
349///
350/// Returns [`StridedError::RankMismatch`] or
351/// [`StridedError::ShapeMismatch`] when the views describe different
352/// extents, and [`StridedError::OffsetOverflow`] when the element count of
353/// the shape does not fit in `usize`. Every check runs before the first
354/// write, so a rejected call leaves `dest` untouched.
355pub fn add_acc<
356    D: Copy + Add<S, Output = D> + Send + Sync,
357    S: Copy + Send + Sync,
358    Op: ElementOp<S>,
359>(
360    dest: &mut StridedViewMut<D>,
361    src: &StridedView<S, Op>,
362    exec: Exec<'_>,
363) -> Result<()> {
364    ensure_same_shape(dest.dims(), src.dims())?;
365
366    let dst_ptr = dest.as_mut_ptr();
367    let src_ptr = src.ptr();
368    let dst_dims = dest.dims();
369    let dst_strides = dest.strides();
370    let src_strides = src.strides();
371
372    if sequential_contiguous_layout(
373        KernelClass::Map,
374        dst_dims,
375        &[dst_strides, src_strides],
376        &exec,
377    )?
378    .is_some()
379    {
380        let len = total_len(dst_dims)?;
381        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
382        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
383        simd::dispatch_if_large(len, || {
384            for i in 0..len {
385                dst[i] = dst[i] + Op::apply(src[i]);
386            }
387        });
388        return Ok(());
389    }
390
391    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
392    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
393
394    let total = total_len(dst_dims)?;
395    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
396        build_plan_fused_small(dst_dims, &strides_list)
397    } else {
398        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
399    };
400
401    let total = total_len(&fused_dims)?;
402    if is_parallel(KernelClass::Map, total, &exec) {
403        let dst_send = SendPtr(dst_ptr);
404        let src_send = SendPtr(src_ptr as *mut S);
405
406        let costs = compute_costs(&ordered_strides);
407        let initial_offsets: crate::inline::OperandVec<isize> =
408            strides_list.iter().map(|_| 0isize).collect();
409        return mapreduce_threaded(
410            &fused_dims,
411            &plan.block,
412            &ordered_strides,
413            &initial_offsets,
414            &costs,
415            KernelClass::Map,
416            exec,
417            &|dims, blocks, strides_list, offsets| {
418                for_each_inner_block_with_offsets(
419                    dims,
420                    blocks,
421                    strides_list,
422                    offsets,
423                    |offsets, len, strides| {
424                        unsafe {
425                            inner_loop_add::<D, S, Op>(
426                                dst_send.as_ptr().offset(offsets[0]),
427                                strides[0],
428                                src_send.as_const().offset(offsets[1]),
429                                strides[1],
430                                len,
431                            )
432                        };
433                        Ok(())
434                    },
435                )
436            },
437        );
438    }
439
440    let initial_offsets: crate::inline::OperandVec<isize> =
441        ordered_strides.iter().map(|_| 0isize).collect();
442    for_each_inner_block_preordered(
443        &fused_dims,
444        &plan.block,
445        &ordered_strides,
446        &initial_offsets,
447        |offsets, len, strides| {
448            unsafe {
449                inner_loop_add::<D, S, Op>(
450                    dst_ptr.offset(offsets[0]),
451                    strides[0],
452                    src_ptr.offset(offsets[1]),
453                    strides[1],
454                    len,
455                )
456            };
457            Ok(())
458        },
459    )
460}
461
462/// Accumulating element-wise multiplication: `dest[i] *= src[i]`.
463///
464/// Source may have a different element type from destination.
465///
466/// # Errors
467///
468/// Returns [`StridedError::RankMismatch`] or
469/// [`StridedError::ShapeMismatch`] when the views describe different
470/// extents, and [`StridedError::OffsetOverflow`] when the element count of
471/// the shape does not fit in `usize`. Every check runs before the first
472/// write, so a rejected call leaves `dest` untouched.
473pub fn mul_acc<
474    D: Copy + Mul<S, Output = D> + Send + Sync,
475    S: Copy + Send + Sync,
476    Op: ElementOp<S>,
477>(
478    dest: &mut StridedViewMut<D>,
479    src: &StridedView<S, Op>,
480    exec: Exec<'_>,
481) -> Result<()> {
482    ensure_same_shape(dest.dims(), src.dims())?;
483
484    let dst_ptr = dest.as_mut_ptr();
485    let src_ptr = src.ptr();
486    let dst_dims = dest.dims();
487    let dst_strides = dest.strides();
488    let src_strides = src.strides();
489
490    if sequential_contiguous_layout(
491        KernelClass::Map,
492        dst_dims,
493        &[dst_strides, src_strides],
494        &exec,
495    )?
496    .is_some()
497    {
498        let len = total_len(dst_dims)?;
499        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
500        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
501        simd::dispatch_if_large(len, || {
502            for i in 0..len {
503                dst[i] = dst[i] * Op::apply(src[i]);
504            }
505        });
506        return Ok(());
507    }
508
509    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
510    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
511
512    let total = total_len(dst_dims)?;
513    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
514        build_plan_fused_small(dst_dims, &strides_list)
515    } else {
516        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
517    };
518
519    let total = total_len(&fused_dims)?;
520    if is_parallel(KernelClass::Map, total, &exec) {
521        let dst_send = SendPtr(dst_ptr);
522        let src_send = SendPtr(src_ptr as *mut S);
523
524        let costs = compute_costs(&ordered_strides);
525        let initial_offsets: crate::inline::OperandVec<isize> =
526            strides_list.iter().map(|_| 0isize).collect();
527        return mapreduce_threaded(
528            &fused_dims,
529            &plan.block,
530            &ordered_strides,
531            &initial_offsets,
532            &costs,
533            KernelClass::Map,
534            exec,
535            &|dims, blocks, strides_list, offsets| {
536                for_each_inner_block_with_offsets(
537                    dims,
538                    blocks,
539                    strides_list,
540                    offsets,
541                    |offsets, len, strides| {
542                        unsafe {
543                            inner_loop_mul::<D, S, Op>(
544                                dst_send.as_ptr().offset(offsets[0]),
545                                strides[0],
546                                src_send.as_const().offset(offsets[1]),
547                                strides[1],
548                                len,
549                            )
550                        };
551                        Ok(())
552                    },
553                )
554            },
555        );
556    }
557
558    let initial_offsets: crate::inline::OperandVec<isize> =
559        ordered_strides.iter().map(|_| 0isize).collect();
560    for_each_inner_block_preordered(
561        &fused_dims,
562        &plan.block,
563        &ordered_strides,
564        &initial_offsets,
565        |offsets, len, strides| {
566            unsafe {
567                inner_loop_mul::<D, S, Op>(
568                    dst_ptr.offset(offsets[0]),
569                    strides[0],
570                    src_ptr.offset(offsets[1]),
571                    strides[1],
572                    len,
573                )
574            };
575            Ok(())
576        },
577    )
578}
579
580/// Accumulating AXPY: `dest[i] = alpha * src[i] + dest[i]`.
581///
582/// Alpha, source, and destination may have different element types.
583///
584/// # Errors
585///
586/// Returns [`StridedError::RankMismatch`] or
587/// [`StridedError::ShapeMismatch`] when the views describe different
588/// extents, and [`StridedError::OffsetOverflow`] when the element count of
589/// the shape does not fit in `usize`. Every check runs before the first
590/// write, so a rejected call leaves `dest` untouched.
591pub fn axpy_acc<D, S, A, Op>(
592    dest: &mut StridedViewMut<D>,
593    src: &StridedView<S, Op>,
594    alpha: A,
595    exec: Exec<'_>,
596) -> Result<()>
597where
598    A: Copy + Mul<S, Output = D> + Sync,
599    D: Copy + Add<D, Output = D> + Send + Sync,
600    S: Copy + Send + Sync,
601    Op: ElementOp<S>,
602{
603    ensure_same_shape(dest.dims(), src.dims())?;
604
605    let dst_ptr = dest.as_mut_ptr();
606    let src_ptr = src.ptr();
607    let dst_dims = dest.dims();
608    let dst_strides = dest.strides();
609    let src_strides = src.strides();
610
611    if sequential_contiguous_layout(
612        KernelClass::Map,
613        dst_dims,
614        &[dst_strides, src_strides],
615        &exec,
616    )?
617    .is_some()
618    {
619        let len = total_len(dst_dims)?;
620        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
621        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
622        simd::dispatch_if_large(len, || {
623            for i in 0..len {
624                dst[i] = alpha * Op::apply(src[i]) + dst[i];
625            }
626        });
627        return Ok(());
628    }
629
630    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
631    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
632
633    let total = total_len(dst_dims)?;
634    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
635        build_plan_fused_small(dst_dims, &strides_list)
636    } else {
637        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
638    };
639
640    let total = total_len(&fused_dims)?;
641    if is_parallel(KernelClass::Map, total, &exec) {
642        let dst_send = SendPtr(dst_ptr);
643        let src_send = SendPtr(src_ptr as *mut S);
644
645        let costs = compute_costs(&ordered_strides);
646        let initial_offsets: crate::inline::OperandVec<isize> =
647            strides_list.iter().map(|_| 0isize).collect();
648        return mapreduce_threaded(
649            &fused_dims,
650            &plan.block,
651            &ordered_strides,
652            &initial_offsets,
653            &costs,
654            KernelClass::Map,
655            exec,
656            &|dims, blocks, strides_list, offsets| {
657                for_each_inner_block_with_offsets(
658                    dims,
659                    blocks,
660                    strides_list,
661                    offsets,
662                    |offsets, len, strides| {
663                        unsafe {
664                            inner_loop_axpy::<D, S, A, Op>(
665                                dst_send.as_ptr().offset(offsets[0]),
666                                strides[0],
667                                src_send.as_const().offset(offsets[1]),
668                                strides[1],
669                                len,
670                                alpha,
671                            )
672                        };
673                        Ok(())
674                    },
675                )
676            },
677        );
678    }
679
680    let initial_offsets: crate::inline::OperandVec<isize> =
681        ordered_strides.iter().map(|_| 0isize).collect();
682    for_each_inner_block_preordered(
683        &fused_dims,
684        &plan.block,
685        &ordered_strides,
686        &initial_offsets,
687        |offsets, len, strides| {
688            unsafe {
689                inner_loop_axpy::<D, S, A, Op>(
690                    dst_ptr.offset(offsets[0]),
691                    strides[0],
692                    src_ptr.offset(offsets[1]),
693                    strides[1],
694                    len,
695                    alpha,
696                )
697            };
698            Ok(())
699        },
700    )
701}
702
703/// Accumulating fused multiply-add: `dest[i] += OpA::apply(a[i]) * OpB::apply(b[i])`.
704///
705/// Operands may have different element types. Element operations are applied lazily.
706///
707/// # Errors
708///
709/// Returns [`StridedError::RankMismatch`] or
710/// [`StridedError::ShapeMismatch`] when `dest` and an operand describe
711/// different extents, and [`StridedError::OffsetOverflow`] when the element
712/// count of the shape does not fit in `usize`. Every check runs before the
713/// first write, so a rejected call leaves `dest` untouched.
714pub fn fma_acc<D, A, B, OpA, OpB>(
715    dest: &mut StridedViewMut<D>,
716    a: &StridedView<A, OpA>,
717    b: &StridedView<B, OpB>,
718    exec: Exec<'_>,
719) -> Result<()>
720where
721    A: Copy + Mul<B, Output = D> + Send + Sync,
722    B: Copy + Send + Sync,
723    D: Copy + Add<D, Output = D> + Send + Sync,
724    OpA: ElementOp<A>,
725    OpB: ElementOp<B>,
726{
727    ensure_same_shape(dest.dims(), a.dims())?;
728    ensure_same_shape(dest.dims(), b.dims())?;
729
730    let dst_ptr = dest.as_mut_ptr();
731    let a_ptr = a.ptr();
732    let b_ptr = b.ptr();
733    let dst_dims = dest.dims();
734    let dst_strides = dest.strides();
735    let a_strides = a.strides();
736    let b_strides = b.strides();
737
738    if sequential_contiguous_layout(
739        KernelClass::Map,
740        dst_dims,
741        &[dst_strides, a_strides, b_strides],
742        &exec,
743    )?
744    .is_some()
745    {
746        let len = total_len(dst_dims)?;
747        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
748        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
749        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
750        simd::dispatch_if_large(len, || {
751            for i in 0..len {
752                dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
753            }
754        });
755        return Ok(());
756    }
757
758    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
759    let elem_size = std::mem::size_of::<D>()
760        .max(std::mem::size_of::<A>())
761        .max(std::mem::size_of::<B>());
762
763    let total = total_len(dst_dims)?;
764    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
765        build_plan_fused_small(dst_dims, &strides_list)
766    } else {
767        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
768    };
769
770    let total = total_len(&fused_dims)?;
771    if is_parallel(KernelClass::Map, total, &exec) {
772        let dst_send = SendPtr(dst_ptr);
773        let a_send = SendPtr(a_ptr as *mut A);
774        let b_send = SendPtr(b_ptr as *mut B);
775
776        let costs = compute_costs(&ordered_strides);
777        let initial_offsets: crate::inline::OperandVec<isize> =
778            strides_list.iter().map(|_| 0isize).collect();
779        return mapreduce_threaded(
780            &fused_dims,
781            &plan.block,
782            &ordered_strides,
783            &initial_offsets,
784            &costs,
785            KernelClass::Map,
786            exec,
787            &|dims, blocks, strides_list, offsets| {
788                for_each_inner_block_with_offsets(
789                    dims,
790                    blocks,
791                    strides_list,
792                    offsets,
793                    |offsets, len, strides| {
794                        unsafe {
795                            inner_loop_fma::<D, A, B, OpA, OpB>(
796                                dst_send.as_ptr().offset(offsets[0]),
797                                strides[0],
798                                a_send.as_const().offset(offsets[1]),
799                                strides[1],
800                                b_send.as_const().offset(offsets[2]),
801                                strides[2],
802                                len,
803                            )
804                        };
805                        Ok(())
806                    },
807                )
808            },
809        );
810    }
811
812    let initial_offsets: crate::inline::OperandVec<isize> =
813        ordered_strides.iter().map(|_| 0isize).collect();
814    for_each_inner_block_preordered(
815        &fused_dims,
816        &plan.block,
817        &ordered_strides,
818        &initial_offsets,
819        |offsets, len, strides| {
820            unsafe {
821                inner_loop_fma::<D, A, B, OpA, OpB>(
822                    dst_ptr.offset(offsets[0]),
823                    strides[0],
824                    a_ptr.offset(offsets[1]),
825                    strides[1],
826                    b_ptr.offset(offsets[2]),
827                    strides[2],
828                    len,
829                )
830            };
831            Ok(())
832        },
833    )
834}
835
836/// Sum all elements: `sum(src)`.
837///
838/// # Evaluation order
839///
840/// The result is independent of the pool when the call runs with a serial
841/// token ([`Exec::serial`]) or a budget of one. With a larger budget a
842/// contiguous identity operand above the threading threshold is summed per
843/// task and the partial sums are combined in a tree, so the result is
844/// repeatable for a fixed budget but can differ between budgets.
845///
846/// Neither path is a strict left to right sum: a contiguous run is folded with
847/// independent SIMD or scalar lanes (see [`reduce`] for the lane scheme of the
848/// generic path). The last bits can therefore differ from a naive loop, and no
849/// bitwise equality across machines or target features is guaranteed.
850///
851/// For a deterministic accumulation of `f32` data in `f64`, use
852/// [`zip_reduce`](crate::zip_reduce) with an `f64` map and a `0.0f64` init.
853///
854/// # Allocation
855///
856/// At rank ≤ [`INLINE_RANK`](strided_view::INLINE_RANK) the view metadata and
857/// the traversal plan are stored inline, so a call performs no heap allocation.
858///
859/// # Errors
860///
861/// Returns [`StridedError::OffsetOverflow`] when the element count of the
862/// view's shape does not fit in `usize`, the only failure this entry can
863/// produce. That check runs first, before any planning or read.
864pub fn sum<
865    T: Copy + Zero + Add<Output = T> + Send + Sync + simd::MaybeSimdOps,
866    Op: ElementOp<T>,
867>(
868    src: &StridedView<T, Op>,
869    exec: Exec<'_>,
870) -> Result<T> {
871    // A zero-element sum is the additive identity. Return before any planning
872    // so an empty view with an enormous stride never reaches block sizing.
873    if total_len(src.dims())? == 0 {
874        return Ok(T::zero());
875    }
876    // SIMD fast path: contiguous Identity view with SIMD support
877    if Op::IS_IDENTITY && same_contiguous_layout(src.dims(), &[src.strides()]).is_some() {
878        let len = total_len(src.dims())?;
879        let src_slice = unsafe { std::slice::from_raw_parts(src.ptr(), len) };
880
881        // The partial sums of the halves are combined in a tree; a type
882        // without SIMD support takes the generic reduction below.
883        if T::try_simd_sum(&[]).is_some() && is_parallel(KernelClass::Reduce, len, &exec) {
884            return Ok(crate::threading::parallel_map_reduce(
885                0..len,
886                crate::cost::for_len_tree(exec, KernelClass::Reduce, len),
887                &|range| T::try_simd_sum(&src_slice[range]).unwrap(),
888                &|left, right| left + right,
889            ));
890        }
891
892        if let Some(result) = T::try_simd_sum(src_slice) {
893            return Ok(result);
894        }
895    }
896    reduce(src, |x| x, |a, b| a + b, T::zero(), exec)
897}
898
899/// Dot product: `sum(OpA::apply(a[i]) * OpB::apply(b[i]))`.
900///
901/// Operands may have different element types. Result type `R` must be `A * B`.
902/// SIMD fast path fires only when `A == B == R` (same type) and both Identity ops.
903///
904/// A widening dot — `f32` operands with an `f64` accumulator — is not
905/// expressible here because `R` is pinned to `A * B`; use
906/// [`zip_reduce`](crate::zip_reduce) with a
907/// `|x, y| x as f64 * y as f64` map instead.
908///
909/// # Evaluation order
910///
911/// This entry takes no parallel branch and no token, so its result does not
912/// depend on any pool: it is computed on the calling thread. The homogeneous contiguous fast path folds with the SIMD
913/// lane scheme of the element type; every other layout folds a single
914/// accumulator along the traversal order of the fused plan, which is left to
915/// right over each unit-stride run. Neither is guaranteed to be bitwise equal
916/// across machines or target features.
917///
918/// # Allocation
919///
920/// At rank ≤ [`INLINE_RANK`](strided_view::INLINE_RANK) the view metadata and
921/// the traversal plan are stored inline, so a call performs no heap allocation.
922///
923/// # Errors
924///
925/// Returns [`StridedError::RankMismatch`] or
926/// [`StridedError::ShapeMismatch`] when the two operands describe different
927/// extents, and [`StridedError::OffsetOverflow`] when the element count of
928/// their shape does not fit in `usize`.
929pub fn dot<A, B, R, OpA, OpB>(a: &StridedView<A, OpA>, b: &StridedView<B, OpB>) -> Result<R>
930where
931    A: Copy + Mul<B, Output = R> + Send + Sync + 'static,
932    B: Copy + Send + Sync + 'static,
933    R: Copy + Zero + Add<Output = R> + Send + Sync + simd::MaybeSimdOps + 'static,
934    OpA: ElementOp<A>,
935    OpB: ElementOp<B>,
936{
937    ensure_same_shape(a.dims(), b.dims())?;
938
939    let a_ptr = a.ptr();
940    let b_ptr = b.ptr();
941    let a_strides = a.strides();
942    let b_strides = b.strides();
943    let a_dims = a.dims();
944    let len = total_len(a_dims)?;
945
946    // A zero-element dot product is the additive identity. Return before any
947    // planning so an empty view with an enormous stride never reaches block
948    // sizing.
949    if len == 0 {
950        return Ok(R::zero());
951    }
952
953    if same_contiguous_layout(a_dims, &[a_strides, b_strides]).is_some() {
954        // SIMD fast path: both contiguous, both Identity ops, same type
955        if OpA::IS_IDENTITY
956            && OpB::IS_IDENTITY
957            && std::any::TypeId::of::<A>() == std::any::TypeId::of::<R>()
958            && std::any::TypeId::of::<B>() == std::any::TypeId::of::<R>()
959        {
960            let sa = unsafe { std::slice::from_raw_parts(a_ptr as *const R, len) };
961            let sb = unsafe { std::slice::from_raw_parts(b_ptr as *const R, len) };
962            if let Some(result) = R::try_simd_dot(sa, sb) {
963                return Ok(result);
964            }
965        }
966
967        // Generic contiguous fast path
968        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
969        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
970        let mut acc = R::zero();
971        simd::dispatch_if_large(len, || {
972            for i in 0..len {
973                acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
974            }
975        });
976        return Ok(acc);
977    }
978
979    let strides_list: [&[isize]; 2] = [a_strides, b_strides];
980    let elem_size = std::mem::size_of::<A>()
981        .max(std::mem::size_of::<B>())
982        .max(std::mem::size_of::<R>());
983
984    let total = total_len(a_dims)?;
985    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
986        build_plan_fused_small(a_dims, &strides_list)
987    } else {
988        build_plan_fused(a_dims, &strides_list, None, elem_size)
989    };
990
991    let mut acc = R::zero();
992    let initial_offsets: crate::inline::OperandVec<isize> =
993        ordered_strides.iter().map(|_| 0isize).collect();
994    for_each_inner_block_preordered(
995        &fused_dims,
996        &plan.block,
997        &ordered_strides,
998        &initial_offsets,
999        |offsets, len, strides| {
1000            acc = unsafe {
1001                inner_loop_dot::<A, B, R, OpA, OpB>(
1002                    a_ptr.offset(offsets[0]),
1003                    strides[0],
1004                    b_ptr.offset(offsets[1]),
1005                    strides[1],
1006                    len,
1007                    acc,
1008                )
1009            };
1010            Ok(())
1011        },
1012    )?;
1013
1014    Ok(acc)
1015}
1016
1017/// Symmetrize a square matrix: `dest = (src + src^T) / 2`.
1018///
1019/// # Errors
1020///
1021/// Returns [`StridedError::RankMismatch`] when `src` is not rank 2,
1022/// [`StridedError::NonSquare`] when its two extents differ, and
1023/// [`StridedError::ScalarConversion`] only when `T::from_f64(0.5)` returns
1024/// `None`. `num_traits::FromPrimitive::from_f64` truncates for the primitive
1025/// integer types (`i32::from_f64(0.5) == Some(0)`), so an integer `T` is not
1026/// rejected: it scales by zero, and the result is zero rather than a half.
1027/// Writing through [`zip_map2_into`](crate::zip_map2_into) adds
1028/// [`StridedError::RankMismatch`] or [`StridedError::ShapeMismatch`] when
1029/// `dest` and the transposed `src` describe different extents, and
1030/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1031/// cannot be proven representable and injective: two logical indices name
1032/// one offset, or the element count or address span is too large to
1033/// represent. Nothing is written unless validation succeeds.
1034pub fn symmetrize_into<T>(
1035    dest: &mut StridedViewMut<T>,
1036    src: &StridedView<T>,
1037    exec: Exec<'_>,
1038) -> Result<()>
1039where
1040    T: Copy
1041        + Add<Output = T>
1042        + Mul<Output = T>
1043        + num_traits::FromPrimitive
1044        + std::ops::Div<Output = T>
1045        + Send
1046        + Sync,
1047{
1048    if src.ndim() != 2 {
1049        return Err(StridedError::RankMismatch(src.ndim(), 2));
1050    }
1051    let rows = src.dims()[0];
1052    let cols = src.dims()[1];
1053    if rows != cols {
1054        return Err(StridedError::NonSquare { rows, cols });
1055    }
1056
1057    let src_t = src.permute(&[1, 0])?;
1058    let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1059
1060    zip_map2_into(dest, src, &src_t, |a, b| (a + b) * half, exec)
1061}
1062
1063/// Conjugate-symmetrize a square matrix: `dest = (src + conj(src^T)) / 2`.
1064///
1065/// # Errors
1066///
1067/// Returns [`StridedError::RankMismatch`] when `src` is not rank 2,
1068/// [`StridedError::NonSquare`] when its two extents differ, and
1069/// [`StridedError::ScalarConversion`] only when `T::from_f64(0.5)` returns
1070/// `None`. `num_traits::FromPrimitive::from_f64` truncates for the primitive
1071/// integer types (`i32::from_f64(0.5) == Some(0)`), so an integer `T` is not
1072/// rejected: it scales by zero, and the result is zero rather than a half.
1073/// Writing through [`zip_map2_into`](crate::zip_map2_into) adds
1074/// [`StridedError::RankMismatch`] or [`StridedError::ShapeMismatch`] when
1075/// `dest` and the adjoint of `src` describe different extents, and
1076/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1077/// cannot be proven representable and injective: two logical indices name
1078/// one offset, or the element count or address span is too large to
1079/// represent. Nothing is written unless validation succeeds.
1080pub fn symmetrize_conj_into<T>(
1081    dest: &mut StridedViewMut<T>,
1082    src: &StridedView<T>,
1083    exec: Exec<'_>,
1084) -> Result<()>
1085where
1086    T: Copy
1087        + ElementOpApply
1088        + Add<Output = T>
1089        + Mul<Output = T>
1090        + num_traits::FromPrimitive
1091        + std::ops::Div<Output = T>
1092        + Send
1093        + Sync,
1094{
1095    if src.ndim() != 2 {
1096        return Err(StridedError::RankMismatch(src.ndim(), 2));
1097    }
1098    let rows = src.dims()[0];
1099    let cols = src.dims()[1];
1100    if rows != cols {
1101        return Err(StridedError::NonSquare { rows, cols });
1102    }
1103
1104    // adjoint = conj + transpose
1105    let src_adj = src.adjoint_2d()?;
1106    let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1107
1108    zip_map2_into(dest, src, &src_adj, |a, b| (a + b) * half, exec)
1109}
1110
1111/// Overwriting copy with scaling: `dest[i] = scale * src[i]`.
1112///
1113/// Scale, source, and destination may have different element types.
1114///
1115/// # Errors
1116///
1117/// Returns [`StridedError::RankMismatch`] or
1118/// [`StridedError::ShapeMismatch`] when the views describe different
1119/// extents, and [`StridedError::NonInjectiveOutputLayout`] when the
1120/// destination layout cannot be proven representable and injective: two
1121/// logical indices name one offset, or the element count or address span is
1122/// too large to represent. Nothing is written unless validation succeeds.
1123pub fn copy_scale_into<D, S, A, Op>(
1124    dest: &mut StridedViewMut<D>,
1125    src: &StridedView<S, Op>,
1126    scale: A,
1127    exec: Exec<'_>,
1128) -> Result<()>
1129where
1130    A: Copy + Mul<S, Output = D> + Sync,
1131    D: Copy + Send + Sync,
1132    S: Copy + Send + Sync,
1133    Op: ElementOp<S>,
1134{
1135    map_into(dest, src, |x| scale * x, exec)
1136}
1137
1138/// Overwriting copy with complex conjugation: `dest[i] = conj(src[i])`.
1139///
1140/// # Errors
1141///
1142/// The rejection set of [`copy_into`]: [`StridedError::RankMismatch`] or
1143/// [`StridedError::ShapeMismatch`] when the views describe different
1144/// extents, [`StridedError::OffsetOverflow`] when the element count of the
1145/// shape does not fit in `usize`, and
1146/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1147/// cannot be proven representable and injective. Nothing is written unless
1148/// validation succeeds.
1149pub fn copy_conj_into<T: Copy + ElementOpApply + Send + Sync>(
1150    dest: &mut StridedViewMut<T>,
1151    src: &StridedView<T>,
1152    exec: Exec<'_>,
1153) -> Result<()> {
1154    let src_conj = src.conj();
1155    copy_into(dest, &src_conj, exec)
1156}
1157
1158#[inline]
1159fn element_transpose_is_identity<T: 'static>() -> bool {
1160    use std::any::TypeId;
1161
1162    macro_rules! matches_type {
1163        ($($ty:ty),* $(,)?) => {{
1164            let id = TypeId::of::<T>();
1165            false $(|| id == TypeId::of::<$ty>())*
1166        }};
1167    }
1168
1169    matches_type!(
1170        f32,
1171        f64,
1172        i8,
1173        i16,
1174        i32,
1175        i64,
1176        i128,
1177        isize,
1178        u8,
1179        u16,
1180        u32,
1181        u64,
1182        u128,
1183        usize,
1184        num_complex::Complex32,
1185        num_complex::Complex64,
1186    )
1187}
1188
1189#[inline]
1190fn element_zero_is_all_bits_zero<T: 'static>() -> bool {
1191    use std::any::TypeId;
1192
1193    macro_rules! matches_type {
1194        ($($ty:ty),* $(,)?) => {{
1195            let id = TypeId::of::<T>();
1196            false $(|| id == TypeId::of::<$ty>())*
1197        }};
1198    }
1199
1200    matches_type!(
1201        f32,
1202        f64,
1203        i8,
1204        i16,
1205        i32,
1206        i64,
1207        i128,
1208        isize,
1209        u8,
1210        u16,
1211        u32,
1212        u64,
1213        u128,
1214        usize,
1215        num_complex::Complex32,
1216        num_complex::Complex64,
1217    )
1218}
1219
1220#[inline]
1221unsafe fn fill_2d<T: Copy + Send + Sync>(
1222    dst: *mut T,
1223    dim0: usize,
1224    dim1: usize,
1225    dst_stride0: isize,
1226    dst_stride1: isize,
1227    value: T,
1228    exec: Exec<'_>,
1229) {
1230    let total = dim0.saturating_mul(dim1);
1231    let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
1232    if !exec.is_serial() {
1233        let dst_send = SendPtr(dst);
1234        if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1235            crate::threading::parallel_for_each(0..dim1, exec, &|columns| {
1236                for j in columns {
1237                    let dst = dst_send.as_ptr();
1238                    unsafe {
1239                        let base = j as isize * dst_stride1;
1240                        for i in 0..dim0 {
1241                            *dst.offset(base + i as isize * dst_stride0) = value;
1242                        }
1243                    }
1244                }
1245            });
1246        } else {
1247            crate::threading::parallel_for_each(0..dim0, exec, &|rows| {
1248                for i in rows {
1249                    let dst = dst_send.as_ptr();
1250                    unsafe {
1251                        let base = i as isize * dst_stride0;
1252                        for j in 0..dim1 {
1253                            *dst.offset(base + j as isize * dst_stride1) = value;
1254                        }
1255                    }
1256                }
1257            });
1258        }
1259        return;
1260    }
1261
1262    if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1263        for j in 0..dim1 {
1264            let base = j as isize * dst_stride1;
1265            for i in 0..dim0 {
1266                *dst.offset(base + i as isize * dst_stride0) = value;
1267            }
1268        }
1269    } else {
1270        for i in 0..dim0 {
1271            let base = i as isize * dst_stride0;
1272            for j in 0..dim1 {
1273                *dst.offset(base + j as isize * dst_stride1) = value;
1274            }
1275        }
1276    }
1277}
1278
1279#[inline]
1280unsafe fn fill_contiguous<T>(dst: *mut T, len: usize, value: T)
1281where
1282    T: Copy + Zero + PartialEq + Send + Sync + 'static,
1283{
1284    if element_zero_is_all_bits_zero::<T>() && value == T::zero() {
1285        std::ptr::write_bytes(dst, 0, len);
1286        return;
1287    }
1288
1289    let dst = std::slice::from_raw_parts_mut(dst, len);
1290    dst.fill(value);
1291}
1292
1293// 4x4 transpose block: both base pointers with both strides, the block index and
1294// the scale.
1295#[allow(clippy::too_many_arguments)]
1296#[inline(always)]
1297unsafe fn transpose_scale_4x4_f64(
1298    dst: *mut f64,
1299    dst_stride0: isize,
1300    dst_stride1: isize,
1301    src: *const f64,
1302    src_stride0: isize,
1303    src_stride1: isize,
1304    i: usize,
1305    j: usize,
1306    scale: f64,
1307) {
1308    let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1309
1310    let s00 = *src_base;
1311    let s10 = *src_base.offset(src_stride0);
1312    let s20 = *src_base.offset(2 * src_stride0);
1313    let s30 = *src_base.offset(3 * src_stride0);
1314
1315    let src_col1 = src_base.offset(src_stride1);
1316    let s01 = *src_col1;
1317    let s11 = *src_col1.offset(src_stride0);
1318    let s21 = *src_col1.offset(2 * src_stride0);
1319    let s31 = *src_col1.offset(3 * src_stride0);
1320
1321    let src_col2 = src_base.offset(2 * src_stride1);
1322    let s02 = *src_col2;
1323    let s12 = *src_col2.offset(src_stride0);
1324    let s22 = *src_col2.offset(2 * src_stride0);
1325    let s32 = *src_col2.offset(3 * src_stride0);
1326
1327    let src_col3 = src_base.offset(3 * src_stride1);
1328    let s03 = *src_col3;
1329    let s13 = *src_col3.offset(src_stride0);
1330    let s23 = *src_col3.offset(2 * src_stride0);
1331    let s33 = *src_col3.offset(3 * src_stride0);
1332
1333    let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1334    *dst_row0 = scale * s00;
1335    *dst_row0.offset(dst_stride0) = scale * s01;
1336    *dst_row0.offset(2 * dst_stride0) = scale * s02;
1337    *dst_row0.offset(3 * dst_stride0) = scale * s03;
1338
1339    let dst_row1 = dst_row0.offset(dst_stride1);
1340    *dst_row1 = scale * s10;
1341    *dst_row1.offset(dst_stride0) = scale * s11;
1342    *dst_row1.offset(2 * dst_stride0) = scale * s12;
1343    *dst_row1.offset(3 * dst_stride0) = scale * s13;
1344
1345    let dst_row2 = dst_row0.offset(2 * dst_stride1);
1346    *dst_row2 = scale * s20;
1347    *dst_row2.offset(dst_stride0) = scale * s21;
1348    *dst_row2.offset(2 * dst_stride0) = scale * s22;
1349    *dst_row2.offset(3 * dst_stride0) = scale * s23;
1350
1351    let dst_row3 = dst_row0.offset(3 * dst_stride1);
1352    *dst_row3 = scale * s30;
1353    *dst_row3.offset(dst_stride0) = scale * s31;
1354    *dst_row3.offset(2 * dst_stride0) = scale * s32;
1355    *dst_row3.offset(3 * dst_stride0) = scale * s33;
1356}
1357
1358#[inline]
1359// Tile kernel for f64: both bases, both stride pairs, the tile bounds, the scale and
1360// the token.
1361#[allow(clippy::too_many_arguments)]
1362unsafe fn copy_transpose_scale_2d_f64_tiled_raw(
1363    dst: *mut f64,
1364    dst_stride0: isize,
1365    dst_stride1: isize,
1366    src: *const f64,
1367    src_stride0: isize,
1368    src_stride1: isize,
1369    src_rows: usize,
1370    src_cols: usize,
1371    scale: f64,
1372    exec: Exec<'_>,
1373) {
1374    const TILE: usize = 4;
1375    let row_full = src_rows / TILE * TILE;
1376    let col_full = src_cols / TILE * TILE;
1377
1378    let total = src_rows.saturating_mul(src_cols);
1379    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1380    if !exec.is_serial() {
1381        let dst_send = SendPtr(dst);
1382        let src_send = SendPtr(src as *mut f64);
1383        let row_tiles = row_full / TILE;
1384        crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1385            for tile_i in tiles {
1386                let i = tile_i * TILE;
1387                let dst = dst_send.as_ptr();
1388                let src = src_send.as_const();
1389                unsafe {
1390                    let mut j = 0;
1391                    while j < col_full {
1392                        transpose_scale_4x4_f64(
1393                            dst,
1394                            dst_stride0,
1395                            dst_stride1,
1396                            src,
1397                            src_stride0,
1398                            src_stride1,
1399                            i,
1400                            j,
1401                            scale,
1402                        );
1403                        j += TILE;
1404                    }
1405                    for j in col_full..src_cols {
1406                        for ii in i..i + TILE {
1407                            *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1408                                scale
1409                                    * *src.offset(
1410                                        ii as isize * src_stride0 + j as isize * src_stride1,
1411                                    );
1412                        }
1413                    }
1414                }
1415            }
1416        });
1417        for i in row_full..src_rows {
1418            for j in 0..src_cols {
1419                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1420                    scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1421            }
1422        }
1423        return;
1424    }
1425
1426    let mut i = 0;
1427    while i < row_full {
1428        let mut j = 0;
1429        while j < col_full {
1430            transpose_scale_4x4_f64(
1431                dst,
1432                dst_stride0,
1433                dst_stride1,
1434                src,
1435                src_stride0,
1436                src_stride1,
1437                i,
1438                j,
1439                scale,
1440            );
1441            j += TILE;
1442        }
1443        for j in col_full..src_cols {
1444            for ii in i..i + TILE {
1445                *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1446                    scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1447            }
1448        }
1449        i += TILE;
1450    }
1451    for i in row_full..src_rows {
1452        for j in 0..src_cols {
1453            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1454                scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1455        }
1456    }
1457}
1458
1459#[inline]
1460#[cfg(test)]
1461unsafe fn try_copy_transpose_scale_2d_f64_tiled(
1462    dest: &mut StridedViewMut<f64>,
1463    src: &StridedView<f64>,
1464    scale: f64,
1465    exec: Exec<'_>,
1466) -> bool {
1467    if src.ndim() != 2 || dest.ndim() != 2 {
1468        return false;
1469    }
1470    let src_dims = src.dims();
1471    if dest.dims() != [src_dims[1], src_dims[0]] {
1472        return false;
1473    }
1474    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1475        return false;
1476    }
1477
1478    copy_transpose_scale_2d_f64_tiled_raw(
1479        dest.as_mut_ptr(),
1480        dest.strides()[0],
1481        dest.strides()[1],
1482        src.ptr(),
1483        src.strides()[0],
1484        src.strides()[1],
1485        src_dims[0],
1486        src_dims[1],
1487        scale,
1488        exec,
1489    );
1490    true
1491}
1492
1493#[inline]
1494unsafe fn try_copy_transpose_scale_2d_f64_tiled_typed<T>(
1495    dest: &mut StridedViewMut<T>,
1496    src: &StridedView<T>,
1497    scale: T,
1498    exec: Exec<'_>,
1499) -> bool
1500where
1501    T: Copy + 'static,
1502{
1503    if std::any::TypeId::of::<T>() != std::any::TypeId::of::<f64>() {
1504        return false;
1505    }
1506
1507    let scale = *(&scale as *const T).cast::<f64>();
1508    if src.ndim() != 2 || dest.ndim() != 2 {
1509        return false;
1510    }
1511    let src_dims = src.dims();
1512    if dest.dims() != [src_dims[1], src_dims[0]] {
1513        return false;
1514    }
1515    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1516        return false;
1517    }
1518
1519    copy_transpose_scale_2d_f64_tiled_raw(
1520        dest.as_mut_ptr().cast::<f64>(),
1521        dest.strides()[0],
1522        dest.strides()[1],
1523        src.ptr().cast::<f64>(),
1524        src.strides()[0],
1525        src.strides()[1],
1526        src_dims[0],
1527        src_dims[1],
1528        scale,
1529        exec,
1530    );
1531    true
1532}
1533
1534// Same 4x4 transpose block for the identity element op, which skips the op.
1535#[allow(clippy::too_many_arguments)]
1536#[inline(always)]
1537unsafe fn transpose_scale_4x4_identity<T>(
1538    dst: *mut T,
1539    dst_stride0: isize,
1540    dst_stride1: isize,
1541    src: *const T,
1542    src_stride0: isize,
1543    src_stride1: isize,
1544    i: usize,
1545    j: usize,
1546    scale: T,
1547) where
1548    T: Copy + Mul<Output = T>,
1549{
1550    let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1551
1552    let s00 = *src_base;
1553    let s10 = *src_base.offset(src_stride0);
1554    let s20 = *src_base.offset(2 * src_stride0);
1555    let s30 = *src_base.offset(3 * src_stride0);
1556
1557    let src_col1 = src_base.offset(src_stride1);
1558    let s01 = *src_col1;
1559    let s11 = *src_col1.offset(src_stride0);
1560    let s21 = *src_col1.offset(2 * src_stride0);
1561    let s31 = *src_col1.offset(3 * src_stride0);
1562
1563    let src_col2 = src_base.offset(2 * src_stride1);
1564    let s02 = *src_col2;
1565    let s12 = *src_col2.offset(src_stride0);
1566    let s22 = *src_col2.offset(2 * src_stride0);
1567    let s32 = *src_col2.offset(3 * src_stride0);
1568
1569    let src_col3 = src_base.offset(3 * src_stride1);
1570    let s03 = *src_col3;
1571    let s13 = *src_col3.offset(src_stride0);
1572    let s23 = *src_col3.offset(2 * src_stride0);
1573    let s33 = *src_col3.offset(3 * src_stride0);
1574
1575    let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1576    *dst_row0 = scale * s00;
1577    *dst_row0.offset(dst_stride0) = scale * s01;
1578    *dst_row0.offset(2 * dst_stride0) = scale * s02;
1579    *dst_row0.offset(3 * dst_stride0) = scale * s03;
1580
1581    let dst_row1 = dst_row0.offset(dst_stride1);
1582    *dst_row1 = scale * s10;
1583    *dst_row1.offset(dst_stride0) = scale * s11;
1584    *dst_row1.offset(2 * dst_stride0) = scale * s12;
1585    *dst_row1.offset(3 * dst_stride0) = scale * s13;
1586
1587    let dst_row2 = dst_row0.offset(2 * dst_stride1);
1588    *dst_row2 = scale * s20;
1589    *dst_row2.offset(dst_stride0) = scale * s21;
1590    *dst_row2.offset(2 * dst_stride0) = scale * s22;
1591    *dst_row2.offset(3 * dst_stride0) = scale * s23;
1592
1593    let dst_row3 = dst_row0.offset(3 * dst_stride1);
1594    *dst_row3 = scale * s30;
1595    *dst_row3.offset(dst_stride0) = scale * s31;
1596    *dst_row3.offset(2 * dst_stride0) = scale * s32;
1597    *dst_row3.offset(3 * dst_stride0) = scale * s33;
1598}
1599
1600#[inline]
1601// Tile kernel: both bases, both stride pairs, the tile bounds, the scale and the token.
1602#[allow(clippy::too_many_arguments)]
1603unsafe fn copy_transpose_scale_2d_identity_tiled_raw<T>(
1604    dst: *mut T,
1605    dst_stride0: isize,
1606    dst_stride1: isize,
1607    src: *const T,
1608    src_stride0: isize,
1609    src_stride1: isize,
1610    src_rows: usize,
1611    src_cols: usize,
1612    scale: T,
1613    exec: Exec<'_>,
1614) where
1615    T: Copy + Mul<Output = T> + Send + Sync,
1616{
1617    const TILE: usize = 4;
1618    let row_full = src_rows / TILE * TILE;
1619    let col_full = src_cols / TILE * TILE;
1620
1621    let total = src_rows.saturating_mul(src_cols);
1622    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1623    if !exec.is_serial() {
1624        let dst_send = SendPtr(dst);
1625        let src_send = SendPtr(src as *mut T);
1626        let row_tiles = row_full / TILE;
1627        crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1628            for tile_i in tiles {
1629                let i = tile_i * TILE;
1630                let dst = dst_send.as_ptr();
1631                let src = src_send.as_const();
1632                unsafe {
1633                    let mut j = 0;
1634                    while j < col_full {
1635                        transpose_scale_4x4_identity(
1636                            dst,
1637                            dst_stride0,
1638                            dst_stride1,
1639                            src,
1640                            src_stride0,
1641                            src_stride1,
1642                            i,
1643                            j,
1644                            scale,
1645                        );
1646                        j += TILE;
1647                    }
1648                    for j in col_full..src_cols {
1649                        for ii in i..i + TILE {
1650                            *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1651                                scale
1652                                    * *src.offset(
1653                                        ii as isize * src_stride0 + j as isize * src_stride1,
1654                                    );
1655                        }
1656                    }
1657                }
1658            }
1659        });
1660        for i in row_full..src_rows {
1661            for j in 0..src_cols {
1662                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1663                    scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1664            }
1665        }
1666        return;
1667    }
1668
1669    let mut i = 0;
1670    while i < row_full {
1671        let mut j = 0;
1672        while j < col_full {
1673            transpose_scale_4x4_identity(
1674                dst,
1675                dst_stride0,
1676                dst_stride1,
1677                src,
1678                src_stride0,
1679                src_stride1,
1680                i,
1681                j,
1682                scale,
1683            );
1684            j += TILE;
1685        }
1686        for j in col_full..src_cols {
1687            for ii in i..i + TILE {
1688                *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1689                    scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1690            }
1691        }
1692        i += TILE;
1693    }
1694    for i in row_full..src_rows {
1695        for j in 0..src_cols {
1696            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1697                scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1698        }
1699    }
1700}
1701
1702#[inline]
1703unsafe fn try_copy_transpose_scale_2d_identity_tiled<T>(
1704    dest: &mut StridedViewMut<T>,
1705    src: &StridedView<T>,
1706    scale: T,
1707    exec: Exec<'_>,
1708) -> bool
1709where
1710    T: Copy + Mul<Output = T> + Send + Sync,
1711{
1712    if src.ndim() != 2 || dest.ndim() != 2 {
1713        return false;
1714    }
1715    let src_dims = src.dims();
1716    if dest.dims() != [src_dims[1], src_dims[0]] {
1717        return false;
1718    }
1719    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1720        return false;
1721    }
1722
1723    copy_transpose_scale_2d_identity_tiled_raw(
1724        dest.as_mut_ptr(),
1725        dest.strides()[0],
1726        dest.strides()[1],
1727        src.ptr(),
1728        src.strides()[0],
1729        src.strides()[1],
1730        src_dims[0],
1731        src_dims[1],
1732        scale,
1733        exec,
1734    );
1735    true
1736}
1737
1738#[inline]
1739// Same signature as the tiled kernels, so the loop and tiled paths stay comparable.
1740#[allow(clippy::too_many_arguments)]
1741unsafe fn copy_transpose_scale_2d_loop<T>(
1742    dst: *mut T,
1743    dst_stride0: isize,
1744    dst_stride1: isize,
1745    src: *const T,
1746    src_stride0: isize,
1747    src_stride1: isize,
1748    src_rows: usize,
1749    src_cols: usize,
1750    scale: T,
1751    exec: Exec<'_>,
1752) where
1753    T: Copy + ElementOpApply + Mul<Output = T> + Send + Sync,
1754{
1755    let total = src_rows.saturating_mul(src_cols);
1756    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1757    if !exec.is_serial() {
1758        let dst_send = SendPtr(dst);
1759        let src_send = SendPtr(src as *mut T);
1760        if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1761            crate::threading::parallel_for_each(0..src_rows, exec, &|rows| {
1762                for i in rows {
1763                    let dst = dst_send.as_ptr();
1764                    let src = src_send.as_const();
1765                    unsafe {
1766                        for j in 0..src_cols {
1767                            let value = (*src
1768                                .offset(i as isize * src_stride0 + j as isize * src_stride1))
1769                            .transpose();
1770                            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1771                                scale * value;
1772                        }
1773                    }
1774                }
1775            });
1776        } else {
1777            crate::threading::parallel_for_each(0..src_cols, exec, &|columns| {
1778                for j in columns {
1779                    let dst = dst_send.as_ptr();
1780                    let src = src_send.as_const();
1781                    unsafe {
1782                        for i in 0..src_rows {
1783                            let value = (*src
1784                                .offset(i as isize * src_stride0 + j as isize * src_stride1))
1785                            .transpose();
1786                            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1787                                scale * value;
1788                        }
1789                    }
1790                }
1791            });
1792        }
1793        return;
1794    }
1795
1796    if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1797        for i in 0..src_rows {
1798            for j in 0..src_cols {
1799                let value =
1800                    (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1801                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1802            }
1803        }
1804    } else {
1805        for j in 0..src_cols {
1806            for i in 0..src_rows {
1807                let value =
1808                    (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1809                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1810            }
1811        }
1812    }
1813}
1814
1815/// Copy with transpose and scaling: `dest[j,i] = scale * src[i,j]`.
1816///
1817/// # Errors
1818///
1819/// Returns [`StridedError::RankMismatch`] when `src` or `dest` is not rank
1820/// 2, [`StridedError::ShapeMismatch`] when `dest` does not have the
1821/// transposed extents of `src`, and
1822/// [`StridedError::OffsetOverflow`] when the element count does not fit in
1823/// `usize`. When `scale` is one and `T`'s transpose is the identity, the
1824/// permutation path also returns [`StridedError::OffsetOverflow`] when the
1825/// layout cannot be expressed in `isize` stride arithmetic or a fused extent
1826/// overflows, and [`StridedError::NonInjectiveOutputLayout`] when the
1827/// destination layout cannot be proven representable and injective.
1828pub fn copy_transpose_scale_into<T>(
1829    dest: &mut StridedViewMut<T>,
1830    src: &StridedView<T>,
1831    scale: T,
1832    exec: Exec<'_>,
1833) -> Result<()>
1834where
1835    T: Copy + ElementOpApply + Mul<Output = T> + Zero + One + PartialEq + Send + Sync + 'static,
1836{
1837    let mut exec = exec;
1838    if src.ndim() != 2 || dest.ndim() != 2 {
1839        return Err(StridedError::RankMismatch(src.ndim(), 2));
1840    }
1841
1842    let src_dims = src.dims();
1843    let expected_dims = [src_dims[1], src_dims[0]];
1844    ensure_same_shape(dest.dims(), &expected_dims)?;
1845    // Reject an element count beyond usize (a huge stride-0 broadcast) up
1846    // front instead of looping over it.
1847    total_len(src_dims)?;
1848
1849    if scale == T::zero() {
1850        unsafe {
1851            if same_contiguous_layout(dest.dims(), &[dest.strides()]).is_some() {
1852                fill_contiguous(dest.as_mut_ptr(), total_len(dest.dims())?, T::zero());
1853            } else {
1854                fill_2d(
1855                    dest.as_mut_ptr(),
1856                    dest.dims()[0],
1857                    dest.dims()[1],
1858                    dest.strides()[0],
1859                    dest.strides()[1],
1860                    T::zero(),
1861                    exec,
1862                );
1863            }
1864        }
1865        return Ok(());
1866    }
1867
1868    let transpose_is_identity = element_transpose_is_identity::<T>();
1869
1870    unsafe {
1871        if transpose_is_identity
1872            && try_copy_transpose_scale_2d_f64_tiled_typed(dest, src, scale, exec.reborrow())
1873        {
1874            return Ok(());
1875        }
1876        if transpose_is_identity
1877            && try_copy_transpose_scale_2d_identity_tiled(dest, src, scale, exec.reborrow())
1878        {
1879            return Ok(());
1880        }
1881    }
1882
1883    if scale == T::one() && transpose_is_identity {
1884        let src_t = src.permute(&[1, 0])?;
1885        return crate::threading::copy_permuted(dest, &src_t, exec);
1886    }
1887
1888    unsafe {
1889        copy_transpose_scale_2d_loop(
1890            dest.as_mut_ptr(),
1891            dest.strides()[0],
1892            dest.strides()[1],
1893            src.ptr(),
1894            src.strides()[0],
1895            src.strides()[1],
1896            src_dims[0],
1897            src_dims[1],
1898            scale,
1899            exec,
1900        );
1901    }
1902    Ok(())
1903}
1904
1905#[cfg(test)]
1906#[path = "ops_view/tests/tiled_tests.rs"]
1907mod tiled_tests;