Skip to main content

strided_basic/
ops_view.rs

1//! High-level operations on dynamic-rank strided views.
2
3use crate::kernel::{
4    build_plan_fused, build_plan_fused_small, ensure_same_shape, for_each_inner_block_preordered,
5    same_contiguous_layout, sequential_contiguous_layout, total_len, SMALL_TENSOR_THRESHOLD,
6};
7use crate::map_view::{copy_map_into, map_into, zip_map2_into};
8use crate::reduce_view::reduce;
9use crate::simd;
10use crate::view::{StridedView, StridedViewMut};
11use crate::{Exec, Result, StridedError};
12use num_traits::{One, Zero};
13use std::mem::MaybeUninit;
14use std::ops::{Add, Mul};
15use strided_view::{ElementOp, ElementOpApply};
16
17use crate::cost::KernelClass;
18use crate::fuse::compute_costs;
19use crate::threading::{
20    for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded, SendPtr,
21};
22// ============================================================================
23// Stride-specialized inner loop helpers for ops_view
24//
25// When all inner strides are 1 (contiguous in the innermost dimension),
26// we use slice-based iteration so LLVM can auto-vectorize effectively.
27// This mirrors the inner_loop_map* helpers in map_view.rs.
28// ============================================================================
29
30// INVARIANT: validated layouts keep all dereferences in bounds. Wrapping
31// advances permit an unused final cursor outside a reversed/gapped allocation.
32/// Inner loop for `add_acc`: `dst[i] += Op::apply(src[i])`.
33#[inline(always)]
34unsafe fn inner_loop_add<D: Copy + Add<S, Output = D>, S: Copy, Op: ElementOp<S>>(
35    dp: *mut D,
36    ds: isize,
37    sp: *const S,
38    ss: isize,
39    len: usize,
40) {
41    if ds == 1 && ss == 1 {
42        let dst = std::slice::from_raw_parts_mut(dp, len);
43        let src = std::slice::from_raw_parts(sp, len);
44        simd::dispatch_if_large(len, || {
45            for i in 0..len {
46                dst[i] = dst[i] + Op::apply(src[i]);
47            }
48        });
49    } else {
50        let mut dp = dp;
51        let mut sp = sp;
52        for _ in 0..len {
53            *dp = *dp + Op::apply(*sp);
54            dp = dp.wrapping_offset(ds);
55            sp = sp.wrapping_offset(ss);
56        }
57    }
58}
59
60/// Inner loop for `mul_acc`: `dst[i] *= Op::apply(src[i])`.
61#[inline(always)]
62unsafe fn inner_loop_mul<D: Copy + Mul<S, Output = D>, S: Copy, Op: ElementOp<S>>(
63    dp: *mut D,
64    ds: isize,
65    sp: *const S,
66    ss: isize,
67    len: usize,
68) {
69    if ds == 1 && ss == 1 {
70        let dst = std::slice::from_raw_parts_mut(dp, len);
71        let src = std::slice::from_raw_parts(sp, len);
72        simd::dispatch_if_large(len, || {
73            for i in 0..len {
74                dst[i] = dst[i] * Op::apply(src[i]);
75            }
76        });
77    } else {
78        let mut dp = dp;
79        let mut sp = sp;
80        for _ in 0..len {
81            *dp = *dp * Op::apply(*sp);
82            dp = dp.wrapping_offset(ds);
83            sp = sp.wrapping_offset(ss);
84        }
85    }
86}
87
88/// Inner loop for `axpy_acc`: `dst[i] = alpha * Op::apply(src[i]) + dst[i]`.
89#[inline(always)]
90unsafe fn inner_loop_axpy<
91    D: Copy + Add<D, Output = D>,
92    S: Copy,
93    A: Copy + Mul<S, Output = D>,
94    Op: ElementOp<S>,
95>(
96    dp: *mut D,
97    ds: isize,
98    sp: *const S,
99    ss: isize,
100    len: usize,
101    alpha: A,
102) {
103    if ds == 1 && ss == 1 {
104        let dst = std::slice::from_raw_parts_mut(dp, len);
105        let src = std::slice::from_raw_parts(sp, len);
106        simd::dispatch_if_large(len, || {
107            for i in 0..len {
108                dst[i] = alpha * Op::apply(src[i]) + dst[i];
109            }
110        });
111    } else {
112        let mut dp = dp;
113        let mut sp = sp;
114        for _ in 0..len {
115            *dp = alpha * Op::apply(*sp) + *dp;
116            dp = dp.wrapping_offset(ds);
117            sp = sp.wrapping_offset(ss);
118        }
119    }
120}
121
122/// Inner loop for `fma_acc`: `dst[i] += OpA::apply(a[i]) * OpB::apply(b[i])`.
123#[inline(always)]
124unsafe fn inner_loop_fma<
125    D: Copy + Add<D, Output = D>,
126    A: Copy + Mul<B, Output = D>,
127    B: Copy,
128    OpA: ElementOp<A>,
129    OpB: ElementOp<B>,
130>(
131    dp: *mut D,
132    ds: isize,
133    ap: *const A,
134    a_s: isize,
135    bp: *const B,
136    b_s: isize,
137    len: usize,
138) {
139    if ds == 1 && a_s == 1 && b_s == 1 {
140        let dst = std::slice::from_raw_parts_mut(dp, len);
141        let sa = std::slice::from_raw_parts(ap, len);
142        let sb = std::slice::from_raw_parts(bp, len);
143        simd::dispatch_if_large(len, || {
144            for i in 0..len {
145                dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
146            }
147        });
148    } else {
149        let mut dp = dp;
150        let mut ap = ap;
151        let mut bp = bp;
152        for _ in 0..len {
153            *dp = *dp + OpA::apply(*ap) * OpB::apply(*bp);
154            dp = dp.wrapping_offset(ds);
155            ap = ap.wrapping_offset(a_s);
156            bp = bp.wrapping_offset(b_s);
157        }
158    }
159}
160
161/// Inner loop for dot: `acc += OpA::apply(a[i]) * OpB::apply(b[i])`.
162#[inline(always)]
163unsafe fn inner_loop_dot<
164    A: Copy + Mul<B, Output = R>,
165    B: Copy,
166    R: Copy + Add<R, Output = R>,
167    OpA: ElementOp<A>,
168    OpB: ElementOp<B>,
169>(
170    ap: *const A,
171    a_s: isize,
172    bp: *const B,
173    b_s: isize,
174    len: usize,
175    mut acc: R,
176) -> R {
177    if a_s == 1 && b_s == 1 {
178        let sa = std::slice::from_raw_parts(ap, len);
179        let sb = std::slice::from_raw_parts(bp, len);
180        simd::dispatch_if_large(len, || {
181            for i in 0..len {
182                acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
183            }
184        });
185    } else {
186        let mut ap = ap;
187        let mut bp = bp;
188        for _ in 0..len {
189            acc = acc + OpA::apply(*ap) * OpB::apply(*bp);
190            ap = ap.wrapping_offset(a_s);
191            bp = bp.wrapping_offset(b_s);
192        }
193    }
194    acc
195}
196
197/// Copy elements from source to destination: `dest[i] = src[i]`.
198///
199/// # Errors
200///
201/// Returns [`StridedError::RankMismatch`] or
202/// [`StridedError::ShapeMismatch`] when the views describe different
203/// extents, and [`StridedError::OffsetOverflow`] when the element count of
204/// the shape does not fit in `usize`. The generic element path also returns
205/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
206/// cannot be proven representable and injective: two logical indices name
207/// one offset, or the address span is too large to represent. Nothing is
208/// written unless validation succeeds.
209pub fn copy_into<T: Copy + Send + Sync, Op: ElementOp<T>>(
210    dest: &mut StridedViewMut<T>,
211    src: &StridedView<T, Op>,
212    exec: Exec<'_>,
213) -> Result<()> {
214    ensure_same_shape(dest.dims(), src.dims())?;
215
216    let dst_ptr = dest.as_mut_ptr();
217    let src_ptr = src.ptr();
218    let dst_dims = dest.dims();
219    let dst_strides = dest.strides();
220    let src_strides = src.strides();
221
222    if sequential_contiguous_layout(
223        KernelClass::Copy,
224        dst_dims,
225        &[dst_strides, src_strides],
226        &exec,
227    )?
228    .is_some()
229    {
230        let len = total_len(dst_dims)?;
231        if Op::IS_IDENTITY {
232            debug_assert!(
233                {
234                    let nbytes = len
235                        .checked_mul(std::mem::size_of::<T>())
236                        .expect("copy size must not overflow");
237                    let dst_start = dst_ptr as usize;
238                    let src_start = src_ptr as usize;
239                    let dst_end = dst_start.saturating_add(nbytes);
240                    let src_end = src_start.saturating_add(nbytes);
241                    dst_end <= src_start || src_end <= dst_start
242                },
243                "overlapping src/dest is not supported"
244            );
245            unsafe { std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len) };
246        } else {
247            let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
248            let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
249            simd::dispatch_if_large(len, || {
250                for i in 0..len {
251                    dst[i] = Op::apply(src[i]);
252                }
253            });
254        }
255        return Ok(());
256    }
257
258    copy_map_into(dest, src, |x| x, exec)
259}
260
261/// Copy initialized values into a potentially uninitialized destination.
262///
263/// On success every logical destination element is initialized; holes are
264/// untouched. Accepts identity views; use `map_into` for element operations
265/// such as conjugation. Equal contiguous f32/f64 layouts and every other type
266/// keep the element map; a non-contiguous f32/f64 copy uses the permutation
267/// engine, split within the budget of `exec`.
268///
269/// # Errors
270/// Returns a shape/rank error for mismatched views, `NonInjectiveOutputLayout`
271/// for overlapping destination elements, or `OffsetOverflow` for size overflow.
272///
273/// # Examples
274/// ```
275/// use std::mem::MaybeUninit;
276/// use strided_basic::{copy_into_uninit, Exec, StridedView, StridedViewMut};
277/// let src = [1.0_f64, 2.0];
278/// let mut dst = [MaybeUninit::uninit(); 2];
279/// copy_into_uninit(
280///     &mut StridedViewMut::new(&mut dst, &[2], &[1], 0).unwrap(),
281///     &StridedView::new(&src, &[2], &[1], 0).unwrap(),
282///     Exec::serial(),
283/// ).unwrap();
284/// // SAFETY: the successful copy initialized both destination elements.
285/// assert_eq!(unsafe { dst[1].assume_init() }, 2.0);
286/// ```
287pub fn copy_into_uninit<T: Copy + Send + Sync + 'static>(
288    dest: &mut StridedViewMut<MaybeUninit<T>>,
289    src: &StridedView<T>,
290    exec: Exec<'_>,
291) -> Result<()> {
292    ensure_same_shape(dest.dims(), src.dims())?;
293    if !crate::layout_check::is_injective_layout(dest.dims(), dest.strides()) {
294        return Err(StridedError::NonInjectiveOutputLayout);
295    }
296    if src.is_empty() {
297        return Ok(());
298    }
299    let _len = src
300        .dims()
301        .iter()
302        .try_fold(1usize, |n, &dim| n.checked_mul(dim))
303        .ok_or(StridedError::OffsetOverflow)?;
304    // The permutation engine's 4/8-byte paths reinterpret storage as native
305    // floats. Restrict that path to those exact types: arbitrary Copy types can
306    // contain uninitialized padding or have weaker alignment (e.g. Complex32).
307    let native_float = std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>()
308        || std::any::TypeId::of::<T>() == std::any::TypeId::of::<f64>();
309    // Equal contiguous layouts go to the element map, whose contiguous path
310    // parallelizes a large copy; the permutation engine has only a serial
311    // contiguous shortcut. Non-contiguous native-float copies use the
312    // permutation engine, serial or parallel.
313    if !native_float
314        || same_contiguous_layout(dest.dims(), &[dest.strides(), src.strides()]).is_some()
315    {
316        return copy_map_into(dest, src, MaybeUninit::new, exec);
317    }
318    // INVARIANT: the permutation engine selects its float paths by the
319    // source element type and only stores into the destination, so an
320    // uninitialized destination with an initialized source is valid.
321    crate::threading::copy_permuted_uninit(dest, src, exec)
322}
323
324/// Copy elements from `src` to `dst`, optimized for col-major destination.
325///
326/// Delegates to the centralized permutation-copy dispatch for the actual work.
327///
328/// # Errors
329///
330/// The rejection set of that dispatch: [`StridedError::RankMismatch`] or
331/// [`StridedError::ShapeMismatch`] when the views describe different
332/// extents, [`StridedError::OffsetOverflow`] when the layout cannot be
333/// expressed in `isize` stride arithmetic or a fused extent overflows, and
334/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
335/// cannot be proven representable and injective. Nothing is written unless
336/// validation succeeds.
337pub fn copy_into_col_major<T: Copy + Send + Sync + 'static>(
338    dst: &mut StridedViewMut<T>,
339    src: &StridedView<T>,
340) -> Result<()> {
341    strided_perm::copy_into_col_major(dst, src, Exec::serial())
342}
343
344/// Accumulating element-wise addition: `dest[i] += src[i]`.
345///
346/// Source may have a different element type from destination.
347///
348/// # Errors
349///
350/// Returns [`StridedError::RankMismatch`] or
351/// [`StridedError::ShapeMismatch`] when the views describe different
352/// extents, and [`StridedError::OffsetOverflow`] when the element count of
353/// the shape does not fit in `usize`. Every check runs before the first
354/// write, so a rejected call leaves `dest` untouched.
355pub fn add_acc<
356    D: Copy + Add<S, Output = D> + Send + Sync,
357    S: Copy + Send + Sync,
358    Op: ElementOp<S>,
359>(
360    dest: &mut StridedViewMut<D>,
361    src: &StridedView<S, Op>,
362    exec: Exec<'_>,
363) -> Result<()> {
364    ensure_same_shape(dest.dims(), src.dims())?;
365
366    let dst_ptr = dest.as_mut_ptr();
367    let src_ptr = src.ptr();
368    let dst_dims = dest.dims();
369    let dst_strides = dest.strides();
370    let src_strides = src.strides();
371
372    if sequential_contiguous_layout(
373        KernelClass::Map,
374        dst_dims,
375        &[dst_strides, src_strides],
376        &exec,
377    )?
378    .is_some()
379    {
380        let len = total_len(dst_dims)?;
381        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
382        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
383        simd::dispatch_if_large(len, || {
384            for i in 0..len {
385                dst[i] = dst[i] + Op::apply(src[i]);
386            }
387        });
388        return Ok(());
389    }
390
391    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
392    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
393
394    let total = total_len(dst_dims)?;
395    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
396        build_plan_fused_small(dst_dims, &strides_list)
397    } else {
398        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
399    };
400
401    let total = total_len(&fused_dims)?;
402    if is_parallel(KernelClass::Map, total, &exec) {
403        let dst_send = SendPtr(dst_ptr);
404        let src_send = SendPtr(src_ptr as *mut S);
405
406        let costs = compute_costs(&ordered_strides);
407        let initial_offsets: crate::inline::OperandVec<isize> =
408            strides_list.iter().map(|_| 0isize).collect();
409        return mapreduce_threaded(
410            &fused_dims,
411            &plan.block,
412            &ordered_strides,
413            &initial_offsets,
414            &costs,
415            KernelClass::Map,
416            exec,
417            &|dims, blocks, strides_list, offsets| {
418                for_each_inner_block_with_offsets(
419                    dims,
420                    blocks,
421                    strides_list,
422                    offsets,
423                    |offsets, len, strides| {
424                        unsafe {
425                            inner_loop_add::<D, S, Op>(
426                                dst_send.as_ptr().offset(offsets[0]),
427                                strides[0],
428                                src_send.as_const().offset(offsets[1]),
429                                strides[1],
430                                len,
431                            )
432                        };
433                        Ok(())
434                    },
435                )
436            },
437        );
438    }
439
440    let initial_offsets: crate::inline::OperandVec<isize> =
441        ordered_strides.iter().map(|_| 0isize).collect();
442    for_each_inner_block_preordered(
443        &fused_dims,
444        &plan.block,
445        &ordered_strides,
446        &initial_offsets,
447        |offsets, len, strides| {
448            unsafe {
449                inner_loop_add::<D, S, Op>(
450                    dst_ptr.offset(offsets[0]),
451                    strides[0],
452                    src_ptr.offset(offsets[1]),
453                    strides[1],
454                    len,
455                )
456            };
457            Ok(())
458        },
459    )
460}
461
462/// Accumulating element-wise multiplication: `dest[i] *= src[i]`.
463///
464/// Source may have a different element type from destination.
465///
466/// # Errors
467///
468/// Returns [`StridedError::RankMismatch`] or
469/// [`StridedError::ShapeMismatch`] when the views describe different
470/// extents, and [`StridedError::OffsetOverflow`] when the element count of
471/// the shape does not fit in `usize`. Every check runs before the first
472/// write, so a rejected call leaves `dest` untouched.
473pub fn mul_acc<
474    D: Copy + Mul<S, Output = D> + Send + Sync,
475    S: Copy + Send + Sync,
476    Op: ElementOp<S>,
477>(
478    dest: &mut StridedViewMut<D>,
479    src: &StridedView<S, Op>,
480    exec: Exec<'_>,
481) -> Result<()> {
482    ensure_same_shape(dest.dims(), src.dims())?;
483
484    let dst_ptr = dest.as_mut_ptr();
485    let src_ptr = src.ptr();
486    let dst_dims = dest.dims();
487    let dst_strides = dest.strides();
488    let src_strides = src.strides();
489
490    if sequential_contiguous_layout(
491        KernelClass::Map,
492        dst_dims,
493        &[dst_strides, src_strides],
494        &exec,
495    )?
496    .is_some()
497    {
498        let len = total_len(dst_dims)?;
499        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
500        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
501        simd::dispatch_if_large(len, || {
502            for i in 0..len {
503                dst[i] = dst[i] * Op::apply(src[i]);
504            }
505        });
506        return Ok(());
507    }
508
509    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
510    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
511
512    let total = total_len(dst_dims)?;
513    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
514        build_plan_fused_small(dst_dims, &strides_list)
515    } else {
516        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
517    };
518
519    let total = total_len(&fused_dims)?;
520    if is_parallel(KernelClass::Map, total, &exec) {
521        let dst_send = SendPtr(dst_ptr);
522        let src_send = SendPtr(src_ptr as *mut S);
523
524        let costs = compute_costs(&ordered_strides);
525        let initial_offsets: crate::inline::OperandVec<isize> =
526            strides_list.iter().map(|_| 0isize).collect();
527        return mapreduce_threaded(
528            &fused_dims,
529            &plan.block,
530            &ordered_strides,
531            &initial_offsets,
532            &costs,
533            KernelClass::Map,
534            exec,
535            &|dims, blocks, strides_list, offsets| {
536                for_each_inner_block_with_offsets(
537                    dims,
538                    blocks,
539                    strides_list,
540                    offsets,
541                    |offsets, len, strides| {
542                        unsafe {
543                            inner_loop_mul::<D, S, Op>(
544                                dst_send.as_ptr().offset(offsets[0]),
545                                strides[0],
546                                src_send.as_const().offset(offsets[1]),
547                                strides[1],
548                                len,
549                            )
550                        };
551                        Ok(())
552                    },
553                )
554            },
555        );
556    }
557
558    let initial_offsets: crate::inline::OperandVec<isize> =
559        ordered_strides.iter().map(|_| 0isize).collect();
560    for_each_inner_block_preordered(
561        &fused_dims,
562        &plan.block,
563        &ordered_strides,
564        &initial_offsets,
565        |offsets, len, strides| {
566            unsafe {
567                inner_loop_mul::<D, S, Op>(
568                    dst_ptr.offset(offsets[0]),
569                    strides[0],
570                    src_ptr.offset(offsets[1]),
571                    strides[1],
572                    len,
573                )
574            };
575            Ok(())
576        },
577    )
578}
579
580/// Accumulating AXPY: `dest[i] = alpha * src[i] + dest[i]`.
581///
582/// Alpha, source, and destination may have different element types.
583///
584/// # Errors
585///
586/// Returns [`StridedError::RankMismatch`] or
587/// [`StridedError::ShapeMismatch`] when the views describe different
588/// extents, and [`StridedError::OffsetOverflow`] when the element count of
589/// the shape does not fit in `usize`. Every check runs before the first
590/// write, so a rejected call leaves `dest` untouched.
591pub fn axpy_acc<D, S, A, Op>(
592    dest: &mut StridedViewMut<D>,
593    src: &StridedView<S, Op>,
594    alpha: A,
595    exec: Exec<'_>,
596) -> Result<()>
597where
598    A: Copy + Mul<S, Output = D> + Sync,
599    D: Copy + Add<D, Output = D> + Send + Sync,
600    S: Copy + Send + Sync,
601    Op: ElementOp<S>,
602{
603    ensure_same_shape(dest.dims(), src.dims())?;
604
605    let dst_ptr = dest.as_mut_ptr();
606    let src_ptr = src.ptr();
607    let dst_dims = dest.dims();
608    let dst_strides = dest.strides();
609    let src_strides = src.strides();
610
611    if sequential_contiguous_layout(
612        KernelClass::Map,
613        dst_dims,
614        &[dst_strides, src_strides],
615        &exec,
616    )?
617    .is_some()
618    {
619        let len = total_len(dst_dims)?;
620        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
621        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
622        simd::dispatch_if_large(len, || {
623            for i in 0..len {
624                dst[i] = alpha * Op::apply(src[i]) + dst[i];
625            }
626        });
627        return Ok(());
628    }
629
630    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
631    let elem_size = std::mem::size_of::<D>().max(std::mem::size_of::<S>());
632
633    let total = total_len(dst_dims)?;
634    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
635        build_plan_fused_small(dst_dims, &strides_list)
636    } else {
637        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
638    };
639
640    let total = total_len(&fused_dims)?;
641    if is_parallel(KernelClass::Map, total, &exec) {
642        let dst_send = SendPtr(dst_ptr);
643        let src_send = SendPtr(src_ptr as *mut S);
644
645        let costs = compute_costs(&ordered_strides);
646        let initial_offsets: crate::inline::OperandVec<isize> =
647            strides_list.iter().map(|_| 0isize).collect();
648        return mapreduce_threaded(
649            &fused_dims,
650            &plan.block,
651            &ordered_strides,
652            &initial_offsets,
653            &costs,
654            KernelClass::Map,
655            exec,
656            &|dims, blocks, strides_list, offsets| {
657                for_each_inner_block_with_offsets(
658                    dims,
659                    blocks,
660                    strides_list,
661                    offsets,
662                    |offsets, len, strides| {
663                        unsafe {
664                            inner_loop_axpy::<D, S, A, Op>(
665                                dst_send.as_ptr().offset(offsets[0]),
666                                strides[0],
667                                src_send.as_const().offset(offsets[1]),
668                                strides[1],
669                                len,
670                                alpha,
671                            )
672                        };
673                        Ok(())
674                    },
675                )
676            },
677        );
678    }
679
680    let initial_offsets: crate::inline::OperandVec<isize> =
681        ordered_strides.iter().map(|_| 0isize).collect();
682    for_each_inner_block_preordered(
683        &fused_dims,
684        &plan.block,
685        &ordered_strides,
686        &initial_offsets,
687        |offsets, len, strides| {
688            unsafe {
689                inner_loop_axpy::<D, S, A, Op>(
690                    dst_ptr.offset(offsets[0]),
691                    strides[0],
692                    src_ptr.offset(offsets[1]),
693                    strides[1],
694                    len,
695                    alpha,
696                )
697            };
698            Ok(())
699        },
700    )
701}
702
703/// Accumulating fused multiply-add: `dest[i] += OpA::apply(a[i]) * OpB::apply(b[i])`.
704///
705/// Operands may have different element types. Element operations are applied lazily.
706///
707/// # Errors
708///
709/// Returns [`StridedError::RankMismatch`] or
710/// [`StridedError::ShapeMismatch`] when `dest` and an operand describe
711/// different extents, and [`StridedError::OffsetOverflow`] when the element
712/// count of the shape does not fit in `usize`. Every check runs before the
713/// first write, so a rejected call leaves `dest` untouched.
714pub fn fma_acc<D, A, B, OpA, OpB>(
715    dest: &mut StridedViewMut<D>,
716    a: &StridedView<A, OpA>,
717    b: &StridedView<B, OpB>,
718    exec: Exec<'_>,
719) -> Result<()>
720where
721    A: Copy + Mul<B, Output = D> + Send + Sync,
722    B: Copy + Send + Sync,
723    D: Copy + Add<D, Output = D> + Send + Sync,
724    OpA: ElementOp<A>,
725    OpB: ElementOp<B>,
726{
727    ensure_same_shape(dest.dims(), a.dims())?;
728    ensure_same_shape(dest.dims(), b.dims())?;
729
730    let dst_ptr = dest.as_mut_ptr();
731    let a_ptr = a.ptr();
732    let b_ptr = b.ptr();
733    let dst_dims = dest.dims();
734    let dst_strides = dest.strides();
735    let a_strides = a.strides();
736    let b_strides = b.strides();
737
738    if sequential_contiguous_layout(
739        KernelClass::Map,
740        dst_dims,
741        &[dst_strides, a_strides, b_strides],
742        &exec,
743    )?
744    .is_some()
745    {
746        let len = total_len(dst_dims)?;
747        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
748        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
749        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
750        simd::dispatch_if_large(len, || {
751            for i in 0..len {
752                dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
753            }
754        });
755        return Ok(());
756    }
757
758    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
759    let elem_size = std::mem::size_of::<D>()
760        .max(std::mem::size_of::<A>())
761        .max(std::mem::size_of::<B>());
762
763    let total = total_len(dst_dims)?;
764    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
765        build_plan_fused_small(dst_dims, &strides_list)
766    } else {
767        build_plan_fused(dst_dims, &strides_list, Some(0), elem_size)
768    };
769
770    let total = total_len(&fused_dims)?;
771    if is_parallel(KernelClass::Map, total, &exec) {
772        let dst_send = SendPtr(dst_ptr);
773        let a_send = SendPtr(a_ptr as *mut A);
774        let b_send = SendPtr(b_ptr as *mut B);
775
776        let costs = compute_costs(&ordered_strides);
777        let initial_offsets: crate::inline::OperandVec<isize> =
778            strides_list.iter().map(|_| 0isize).collect();
779        return mapreduce_threaded(
780            &fused_dims,
781            &plan.block,
782            &ordered_strides,
783            &initial_offsets,
784            &costs,
785            KernelClass::Map,
786            exec,
787            &|dims, blocks, strides_list, offsets| {
788                for_each_inner_block_with_offsets(
789                    dims,
790                    blocks,
791                    strides_list,
792                    offsets,
793                    |offsets, len, strides| {
794                        unsafe {
795                            inner_loop_fma::<D, A, B, OpA, OpB>(
796                                dst_send.as_ptr().offset(offsets[0]),
797                                strides[0],
798                                a_send.as_const().offset(offsets[1]),
799                                strides[1],
800                                b_send.as_const().offset(offsets[2]),
801                                strides[2],
802                                len,
803                            )
804                        };
805                        Ok(())
806                    },
807                )
808            },
809        );
810    }
811
812    let initial_offsets: crate::inline::OperandVec<isize> =
813        ordered_strides.iter().map(|_| 0isize).collect();
814    for_each_inner_block_preordered(
815        &fused_dims,
816        &plan.block,
817        &ordered_strides,
818        &initial_offsets,
819        |offsets, len, strides| {
820            unsafe {
821                inner_loop_fma::<D, A, B, OpA, OpB>(
822                    dst_ptr.offset(offsets[0]),
823                    strides[0],
824                    a_ptr.offset(offsets[1]),
825                    strides[1],
826                    b_ptr.offset(offsets[2]),
827                    strides[2],
828                    len,
829                )
830            };
831            Ok(())
832        },
833    )
834}
835
836/// Sum all elements: `sum(src)`.
837///
838/// # Evaluation order
839///
840/// The result is independent of the pool when the call runs with a serial
841/// token ([`Exec::serial`]) or a budget of one. With a larger budget a
842/// contiguous identity operand above the threading threshold is summed per
843/// task and the partial sums are combined in a tree, so the result is
844/// repeatable for a fixed budget but can differ between budgets.
845///
846/// Neither path is a strict left to right sum: a contiguous run is folded with
847/// independent SIMD or scalar lanes (see [`reduce`] for the lane scheme of the
848/// generic path). The last bits can therefore differ from a naive loop, and no
849/// bitwise equality across machines or target features is guaranteed.
850///
851/// For a deterministic accumulation of `f32` data in `f64`, use
852/// [`zip_reduce`](crate::zip_reduce) with an `f64` map and a `0.0f64` init.
853///
854/// # Allocation
855///
856/// At rank ≤ [`INLINE_RANK`](strided_view::INLINE_RANK) the view metadata and
857/// the traversal plan are stored inline, so a call performs no heap allocation.
858///
859/// # Errors
860///
861/// Returns [`StridedError::OffsetOverflow`] when the element count of the
862/// view's shape does not fit in `usize`, the only failure this entry can
863/// produce. That check runs first, before any planning or read.
864pub fn sum<
865    T: Copy + Zero + Add<Output = T> + Send + Sync + simd::MaybeSimdOps,
866    Op: ElementOp<T>,
867>(
868    src: &StridedView<T, Op>,
869    exec: Exec<'_>,
870) -> Result<T> {
871    // A zero-element sum is the additive identity. Return before any planning
872    // so an empty view with an enormous stride never reaches block sizing.
873    if total_len(src.dims())? == 0 {
874        return Ok(T::zero());
875    }
876    // SIMD fast path: contiguous Identity view with SIMD support
877    if Op::IS_IDENTITY {
878        if same_contiguous_layout(src.dims(), &[src.strides()]).is_some() {
879            let len = total_len(src.dims())?;
880            let src_slice = unsafe { std::slice::from_raw_parts(src.ptr(), len) };
881
882            // The partial sums of the halves are combined in a tree; a type
883            // without SIMD support takes the generic reduction below.
884            if T::try_simd_sum(&[]).is_some() && is_parallel(KernelClass::Reduce, len, &exec) {
885                return Ok(crate::threading::parallel_map_reduce(
886                    0..len,
887                    crate::cost::for_len_tree(exec, KernelClass::Reduce, len),
888                    &|range| T::try_simd_sum(&src_slice[range]).unwrap(),
889                    &|left, right| left + right,
890                ));
891            }
892
893            if let Some(result) = T::try_simd_sum(src_slice) {
894                return Ok(result);
895            }
896        }
897    }
898    reduce(src, |x| x, |a, b| a + b, T::zero(), exec)
899}
900
901/// Dot product: `sum(OpA::apply(a[i]) * OpB::apply(b[i]))`.
902///
903/// Operands may have different element types. Result type `R` must be `A * B`.
904/// SIMD fast path fires only when `A == B == R` (same type) and both Identity ops.
905///
906/// A widening dot — `f32` operands with an `f64` accumulator — is not
907/// expressible here because `R` is pinned to `A * B`; use
908/// [`zip_reduce`](crate::zip_reduce) with a
909/// `|x, y| x as f64 * y as f64` map instead.
910///
911/// # Evaluation order
912///
913/// This entry takes no parallel branch and no token, so its result does not
914/// depend on any pool: it is computed on the calling thread. The homogeneous contiguous fast path folds with the SIMD
915/// lane scheme of the element type; every other layout folds a single
916/// accumulator along the traversal order of the fused plan, which is left to
917/// right over each unit-stride run. Neither is guaranteed to be bitwise equal
918/// across machines or target features.
919///
920/// # Allocation
921///
922/// At rank ≤ [`INLINE_RANK`](strided_view::INLINE_RANK) the view metadata and
923/// the traversal plan are stored inline, so a call performs no heap allocation.
924///
925/// # Errors
926///
927/// Returns [`StridedError::RankMismatch`] or
928/// [`StridedError::ShapeMismatch`] when the two operands describe different
929/// extents, and [`StridedError::OffsetOverflow`] when the element count of
930/// their shape does not fit in `usize`.
931pub fn dot<A, B, R, OpA, OpB>(a: &StridedView<A, OpA>, b: &StridedView<B, OpB>) -> Result<R>
932where
933    A: Copy + Mul<B, Output = R> + Send + Sync + 'static,
934    B: Copy + Send + Sync + 'static,
935    R: Copy + Zero + Add<Output = R> + Send + Sync + simd::MaybeSimdOps + 'static,
936    OpA: ElementOp<A>,
937    OpB: ElementOp<B>,
938{
939    ensure_same_shape(a.dims(), b.dims())?;
940
941    let a_ptr = a.ptr();
942    let b_ptr = b.ptr();
943    let a_strides = a.strides();
944    let b_strides = b.strides();
945    let a_dims = a.dims();
946    let len = total_len(a_dims)?;
947
948    // A zero-element dot product is the additive identity. Return before any
949    // planning so an empty view with an enormous stride never reaches block
950    // sizing.
951    if len == 0 {
952        return Ok(R::zero());
953    }
954
955    if same_contiguous_layout(a_dims, &[a_strides, b_strides]).is_some() {
956        // SIMD fast path: both contiguous, both Identity ops, same type
957        if OpA::IS_IDENTITY
958            && OpB::IS_IDENTITY
959            && std::any::TypeId::of::<A>() == std::any::TypeId::of::<R>()
960            && std::any::TypeId::of::<B>() == std::any::TypeId::of::<R>()
961        {
962            let sa = unsafe { std::slice::from_raw_parts(a_ptr as *const R, len) };
963            let sb = unsafe { std::slice::from_raw_parts(b_ptr as *const R, len) };
964            if let Some(result) = R::try_simd_dot(sa, sb) {
965                return Ok(result);
966            }
967        }
968
969        // Generic contiguous fast path
970        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
971        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
972        let mut acc = R::zero();
973        simd::dispatch_if_large(len, || {
974            for i in 0..len {
975                acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
976            }
977        });
978        return Ok(acc);
979    }
980
981    let strides_list: [&[isize]; 2] = [a_strides, b_strides];
982    let elem_size = std::mem::size_of::<A>()
983        .max(std::mem::size_of::<B>())
984        .max(std::mem::size_of::<R>());
985
986    let total = total_len(a_dims)?;
987    let (fused_dims, ordered_strides, plan) = if total <= SMALL_TENSOR_THRESHOLD {
988        build_plan_fused_small(a_dims, &strides_list)
989    } else {
990        build_plan_fused(a_dims, &strides_list, None, elem_size)
991    };
992
993    let mut acc = R::zero();
994    let initial_offsets: crate::inline::OperandVec<isize> =
995        ordered_strides.iter().map(|_| 0isize).collect();
996    for_each_inner_block_preordered(
997        &fused_dims,
998        &plan.block,
999        &ordered_strides,
1000        &initial_offsets,
1001        |offsets, len, strides| {
1002            acc = unsafe {
1003                inner_loop_dot::<A, B, R, OpA, OpB>(
1004                    a_ptr.offset(offsets[0]),
1005                    strides[0],
1006                    b_ptr.offset(offsets[1]),
1007                    strides[1],
1008                    len,
1009                    acc,
1010                )
1011            };
1012            Ok(())
1013        },
1014    )?;
1015
1016    Ok(acc)
1017}
1018
1019/// Symmetrize a square matrix: `dest = (src + src^T) / 2`.
1020///
1021/// # Errors
1022///
1023/// Returns [`StridedError::RankMismatch`] when `src` is not rank 2,
1024/// [`StridedError::NonSquare`] when its two extents differ, and
1025/// [`StridedError::ScalarConversion`] only when `T::from_f64(0.5)` returns
1026/// `None`. `num_traits::FromPrimitive::from_f64` truncates for the primitive
1027/// integer types (`i32::from_f64(0.5) == Some(0)`), so an integer `T` is not
1028/// rejected: it scales by zero, and the result is zero rather than a half.
1029/// Writing through [`zip_map2_into`](crate::zip_map2_into) adds
1030/// [`StridedError::RankMismatch`] or [`StridedError::ShapeMismatch`] when
1031/// `dest` and the transposed `src` describe different extents, and
1032/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1033/// cannot be proven representable and injective: two logical indices name
1034/// one offset, or the element count or address span is too large to
1035/// represent. Nothing is written unless validation succeeds.
1036pub fn symmetrize_into<T>(
1037    dest: &mut StridedViewMut<T>,
1038    src: &StridedView<T>,
1039    exec: Exec<'_>,
1040) -> Result<()>
1041where
1042    T: Copy
1043        + Add<Output = T>
1044        + Mul<Output = T>
1045        + num_traits::FromPrimitive
1046        + std::ops::Div<Output = T>
1047        + Send
1048        + Sync,
1049{
1050    if src.ndim() != 2 {
1051        return Err(StridedError::RankMismatch(src.ndim(), 2));
1052    }
1053    let rows = src.dims()[0];
1054    let cols = src.dims()[1];
1055    if rows != cols {
1056        return Err(StridedError::NonSquare { rows, cols });
1057    }
1058
1059    let src_t = src.permute(&[1, 0])?;
1060    let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1061
1062    zip_map2_into(dest, src, &src_t, |a, b| (a + b) * half, exec)
1063}
1064
1065/// Conjugate-symmetrize a square matrix: `dest = (src + conj(src^T)) / 2`.
1066///
1067/// # Errors
1068///
1069/// Returns [`StridedError::RankMismatch`] when `src` is not rank 2,
1070/// [`StridedError::NonSquare`] when its two extents differ, and
1071/// [`StridedError::ScalarConversion`] only when `T::from_f64(0.5)` returns
1072/// `None`. `num_traits::FromPrimitive::from_f64` truncates for the primitive
1073/// integer types (`i32::from_f64(0.5) == Some(0)`), so an integer `T` is not
1074/// rejected: it scales by zero, and the result is zero rather than a half.
1075/// Writing through [`zip_map2_into`](crate::zip_map2_into) adds
1076/// [`StridedError::RankMismatch`] or [`StridedError::ShapeMismatch`] when
1077/// `dest` and the adjoint of `src` describe different extents, and
1078/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1079/// cannot be proven representable and injective: two logical indices name
1080/// one offset, or the element count or address span is too large to
1081/// represent. Nothing is written unless validation succeeds.
1082pub fn symmetrize_conj_into<T>(
1083    dest: &mut StridedViewMut<T>,
1084    src: &StridedView<T>,
1085    exec: Exec<'_>,
1086) -> Result<()>
1087where
1088    T: Copy
1089        + ElementOpApply
1090        + Add<Output = T>
1091        + Mul<Output = T>
1092        + num_traits::FromPrimitive
1093        + std::ops::Div<Output = T>
1094        + Send
1095        + Sync,
1096{
1097    if src.ndim() != 2 {
1098        return Err(StridedError::RankMismatch(src.ndim(), 2));
1099    }
1100    let rows = src.dims()[0];
1101    let cols = src.dims()[1];
1102    if rows != cols {
1103        return Err(StridedError::NonSquare { rows, cols });
1104    }
1105
1106    // adjoint = conj + transpose
1107    let src_adj = src.adjoint_2d()?;
1108    let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1109
1110    zip_map2_into(dest, src, &src_adj, |a, b| (a + b) * half, exec)
1111}
1112
1113/// Overwriting copy with scaling: `dest[i] = scale * src[i]`.
1114///
1115/// Scale, source, and destination may have different element types.
1116///
1117/// # Errors
1118///
1119/// Returns [`StridedError::RankMismatch`] or
1120/// [`StridedError::ShapeMismatch`] when the views describe different
1121/// extents, and [`StridedError::NonInjectiveOutputLayout`] when the
1122/// destination layout cannot be proven representable and injective: two
1123/// logical indices name one offset, or the element count or address span is
1124/// too large to represent. Nothing is written unless validation succeeds.
1125pub fn copy_scale_into<D, S, A, Op>(
1126    dest: &mut StridedViewMut<D>,
1127    src: &StridedView<S, Op>,
1128    scale: A,
1129    exec: Exec<'_>,
1130) -> Result<()>
1131where
1132    A: Copy + Mul<S, Output = D> + Sync,
1133    D: Copy + Send + Sync,
1134    S: Copy + Send + Sync,
1135    Op: ElementOp<S>,
1136{
1137    map_into(dest, src, |x| scale * x, exec)
1138}
1139
1140/// Overwriting copy with complex conjugation: `dest[i] = conj(src[i])`.
1141///
1142/// # Errors
1143///
1144/// The rejection set of [`copy_into`]: [`StridedError::RankMismatch`] or
1145/// [`StridedError::ShapeMismatch`] when the views describe different
1146/// extents, [`StridedError::OffsetOverflow`] when the element count of the
1147/// shape does not fit in `usize`, and
1148/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1149/// cannot be proven representable and injective. Nothing is written unless
1150/// validation succeeds.
1151pub fn copy_conj_into<T: Copy + ElementOpApply + Send + Sync>(
1152    dest: &mut StridedViewMut<T>,
1153    src: &StridedView<T>,
1154    exec: Exec<'_>,
1155) -> Result<()> {
1156    let src_conj = src.conj();
1157    copy_into(dest, &src_conj, exec)
1158}
1159
1160#[inline]
1161fn element_transpose_is_identity<T: 'static>() -> bool {
1162    use std::any::TypeId;
1163
1164    macro_rules! matches_type {
1165        ($($ty:ty),* $(,)?) => {{
1166            let id = TypeId::of::<T>();
1167            false $(|| id == TypeId::of::<$ty>())*
1168        }};
1169    }
1170
1171    matches_type!(
1172        f32,
1173        f64,
1174        i8,
1175        i16,
1176        i32,
1177        i64,
1178        i128,
1179        isize,
1180        u8,
1181        u16,
1182        u32,
1183        u64,
1184        u128,
1185        usize,
1186        num_complex::Complex32,
1187        num_complex::Complex64,
1188    )
1189}
1190
1191#[inline]
1192fn element_zero_is_all_bits_zero<T: 'static>() -> bool {
1193    use std::any::TypeId;
1194
1195    macro_rules! matches_type {
1196        ($($ty:ty),* $(,)?) => {{
1197            let id = TypeId::of::<T>();
1198            false $(|| id == TypeId::of::<$ty>())*
1199        }};
1200    }
1201
1202    matches_type!(
1203        f32,
1204        f64,
1205        i8,
1206        i16,
1207        i32,
1208        i64,
1209        i128,
1210        isize,
1211        u8,
1212        u16,
1213        u32,
1214        u64,
1215        u128,
1216        usize,
1217        num_complex::Complex32,
1218        num_complex::Complex64,
1219    )
1220}
1221
1222#[inline]
1223unsafe fn fill_2d<T: Copy + Send + Sync>(
1224    dst: *mut T,
1225    dim0: usize,
1226    dim1: usize,
1227    dst_stride0: isize,
1228    dst_stride1: isize,
1229    value: T,
1230    exec: Exec<'_>,
1231) {
1232    let total = dim0.saturating_mul(dim1);
1233    let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
1234    if !exec.is_serial() {
1235        let dst_send = SendPtr(dst);
1236        if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1237            crate::threading::parallel_for_each(0..dim1, exec, &|columns| {
1238                for j in columns {
1239                    let dst = dst_send.as_ptr();
1240                    unsafe {
1241                        let base = j as isize * dst_stride1;
1242                        for i in 0..dim0 {
1243                            *dst.offset(base + i as isize * dst_stride0) = value;
1244                        }
1245                    }
1246                }
1247            });
1248        } else {
1249            crate::threading::parallel_for_each(0..dim0, exec, &|rows| {
1250                for i in rows {
1251                    let dst = dst_send.as_ptr();
1252                    unsafe {
1253                        let base = i as isize * dst_stride0;
1254                        for j in 0..dim1 {
1255                            *dst.offset(base + j as isize * dst_stride1) = value;
1256                        }
1257                    }
1258                }
1259            });
1260        }
1261        return;
1262    }
1263
1264    if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1265        for j in 0..dim1 {
1266            let base = j as isize * dst_stride1;
1267            for i in 0..dim0 {
1268                *dst.offset(base + i as isize * dst_stride0) = value;
1269            }
1270        }
1271    } else {
1272        for i in 0..dim0 {
1273            let base = i as isize * dst_stride0;
1274            for j in 0..dim1 {
1275                *dst.offset(base + j as isize * dst_stride1) = value;
1276            }
1277        }
1278    }
1279}
1280
1281#[inline]
1282unsafe fn fill_contiguous<T>(dst: *mut T, len: usize, value: T)
1283where
1284    T: Copy + Zero + PartialEq + Send + Sync + 'static,
1285{
1286    if element_zero_is_all_bits_zero::<T>() && value == T::zero() {
1287        std::ptr::write_bytes(dst, 0, len);
1288        return;
1289    }
1290
1291    let dst = std::slice::from_raw_parts_mut(dst, len);
1292    dst.fill(value);
1293}
1294
1295#[inline(always)]
1296unsafe fn transpose_scale_4x4_f64(
1297    dst: *mut f64,
1298    dst_stride0: isize,
1299    dst_stride1: isize,
1300    src: *const f64,
1301    src_stride0: isize,
1302    src_stride1: isize,
1303    i: usize,
1304    j: usize,
1305    scale: f64,
1306) {
1307    let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1308
1309    let s00 = *src_base;
1310    let s10 = *src_base.offset(src_stride0);
1311    let s20 = *src_base.offset(2 * src_stride0);
1312    let s30 = *src_base.offset(3 * src_stride0);
1313
1314    let src_col1 = src_base.offset(src_stride1);
1315    let s01 = *src_col1;
1316    let s11 = *src_col1.offset(src_stride0);
1317    let s21 = *src_col1.offset(2 * src_stride0);
1318    let s31 = *src_col1.offset(3 * src_stride0);
1319
1320    let src_col2 = src_base.offset(2 * src_stride1);
1321    let s02 = *src_col2;
1322    let s12 = *src_col2.offset(src_stride0);
1323    let s22 = *src_col2.offset(2 * src_stride0);
1324    let s32 = *src_col2.offset(3 * src_stride0);
1325
1326    let src_col3 = src_base.offset(3 * src_stride1);
1327    let s03 = *src_col3;
1328    let s13 = *src_col3.offset(src_stride0);
1329    let s23 = *src_col3.offset(2 * src_stride0);
1330    let s33 = *src_col3.offset(3 * src_stride0);
1331
1332    let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1333    *dst_row0 = scale * s00;
1334    *dst_row0.offset(dst_stride0) = scale * s01;
1335    *dst_row0.offset(2 * dst_stride0) = scale * s02;
1336    *dst_row0.offset(3 * dst_stride0) = scale * s03;
1337
1338    let dst_row1 = dst_row0.offset(dst_stride1);
1339    *dst_row1 = scale * s10;
1340    *dst_row1.offset(dst_stride0) = scale * s11;
1341    *dst_row1.offset(2 * dst_stride0) = scale * s12;
1342    *dst_row1.offset(3 * dst_stride0) = scale * s13;
1343
1344    let dst_row2 = dst_row0.offset(2 * dst_stride1);
1345    *dst_row2 = scale * s20;
1346    *dst_row2.offset(dst_stride0) = scale * s21;
1347    *dst_row2.offset(2 * dst_stride0) = scale * s22;
1348    *dst_row2.offset(3 * dst_stride0) = scale * s23;
1349
1350    let dst_row3 = dst_row0.offset(3 * dst_stride1);
1351    *dst_row3 = scale * s30;
1352    *dst_row3.offset(dst_stride0) = scale * s31;
1353    *dst_row3.offset(2 * dst_stride0) = scale * s32;
1354    *dst_row3.offset(3 * dst_stride0) = scale * s33;
1355}
1356
1357#[inline]
1358#[allow(clippy::too_many_arguments)]
1359unsafe fn copy_transpose_scale_2d_f64_tiled_raw(
1360    dst: *mut f64,
1361    dst_stride0: isize,
1362    dst_stride1: isize,
1363    src: *const f64,
1364    src_stride0: isize,
1365    src_stride1: isize,
1366    src_rows: usize,
1367    src_cols: usize,
1368    scale: f64,
1369    exec: Exec<'_>,
1370) {
1371    const TILE: usize = 4;
1372    let row_full = src_rows / TILE * TILE;
1373    let col_full = src_cols / TILE * TILE;
1374
1375    let total = src_rows.saturating_mul(src_cols);
1376    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1377    if !exec.is_serial() {
1378        let dst_send = SendPtr(dst);
1379        let src_send = SendPtr(src as *mut f64);
1380        let row_tiles = row_full / TILE;
1381        crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1382            for tile_i in tiles {
1383                let i = tile_i * TILE;
1384                let dst = dst_send.as_ptr();
1385                let src = src_send.as_const();
1386                unsafe {
1387                    let mut j = 0;
1388                    while j < col_full {
1389                        transpose_scale_4x4_f64(
1390                            dst,
1391                            dst_stride0,
1392                            dst_stride1,
1393                            src,
1394                            src_stride0,
1395                            src_stride1,
1396                            i,
1397                            j,
1398                            scale,
1399                        );
1400                        j += TILE;
1401                    }
1402                    for j in col_full..src_cols {
1403                        for ii in i..i + TILE {
1404                            *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1405                                scale
1406                                    * *src.offset(
1407                                        ii as isize * src_stride0 + j as isize * src_stride1,
1408                                    );
1409                        }
1410                    }
1411                }
1412            }
1413        });
1414        for i in row_full..src_rows {
1415            for j in 0..src_cols {
1416                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1417                    scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1418            }
1419        }
1420        return;
1421    }
1422
1423    let mut i = 0;
1424    while i < row_full {
1425        let mut j = 0;
1426        while j < col_full {
1427            transpose_scale_4x4_f64(
1428                dst,
1429                dst_stride0,
1430                dst_stride1,
1431                src,
1432                src_stride0,
1433                src_stride1,
1434                i,
1435                j,
1436                scale,
1437            );
1438            j += TILE;
1439        }
1440        for j in col_full..src_cols {
1441            for ii in i..i + TILE {
1442                *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1443                    scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1444            }
1445        }
1446        i += TILE;
1447    }
1448    for i in row_full..src_rows {
1449        for j in 0..src_cols {
1450            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1451                scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1452        }
1453    }
1454}
1455
1456#[inline]
1457#[cfg(test)]
1458unsafe fn try_copy_transpose_scale_2d_f64_tiled(
1459    dest: &mut StridedViewMut<f64>,
1460    src: &StridedView<f64>,
1461    scale: f64,
1462    exec: Exec<'_>,
1463) -> bool {
1464    if src.ndim() != 2 || dest.ndim() != 2 {
1465        return false;
1466    }
1467    let src_dims = src.dims();
1468    if dest.dims() != [src_dims[1], src_dims[0]] {
1469        return false;
1470    }
1471    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1472        return false;
1473    }
1474
1475    copy_transpose_scale_2d_f64_tiled_raw(
1476        dest.as_mut_ptr(),
1477        dest.strides()[0],
1478        dest.strides()[1],
1479        src.ptr(),
1480        src.strides()[0],
1481        src.strides()[1],
1482        src_dims[0],
1483        src_dims[1],
1484        scale,
1485        exec,
1486    );
1487    true
1488}
1489
1490#[inline]
1491unsafe fn try_copy_transpose_scale_2d_f64_tiled_typed<T>(
1492    dest: &mut StridedViewMut<T>,
1493    src: &StridedView<T>,
1494    scale: T,
1495    exec: Exec<'_>,
1496) -> bool
1497where
1498    T: Copy + 'static,
1499{
1500    if std::any::TypeId::of::<T>() != std::any::TypeId::of::<f64>() {
1501        return false;
1502    }
1503
1504    let scale = *(&scale as *const T).cast::<f64>();
1505    if src.ndim() != 2 || dest.ndim() != 2 {
1506        return false;
1507    }
1508    let src_dims = src.dims();
1509    if dest.dims() != [src_dims[1], src_dims[0]] {
1510        return false;
1511    }
1512    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1513        return false;
1514    }
1515
1516    copy_transpose_scale_2d_f64_tiled_raw(
1517        dest.as_mut_ptr().cast::<f64>(),
1518        dest.strides()[0],
1519        dest.strides()[1],
1520        src.ptr().cast::<f64>(),
1521        src.strides()[0],
1522        src.strides()[1],
1523        src_dims[0],
1524        src_dims[1],
1525        scale,
1526        exec,
1527    );
1528    true
1529}
1530
1531#[inline(always)]
1532unsafe fn transpose_scale_4x4_identity<T>(
1533    dst: *mut T,
1534    dst_stride0: isize,
1535    dst_stride1: isize,
1536    src: *const T,
1537    src_stride0: isize,
1538    src_stride1: isize,
1539    i: usize,
1540    j: usize,
1541    scale: T,
1542) where
1543    T: Copy + Mul<Output = T>,
1544{
1545    let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1546
1547    let s00 = *src_base;
1548    let s10 = *src_base.offset(src_stride0);
1549    let s20 = *src_base.offset(2 * src_stride0);
1550    let s30 = *src_base.offset(3 * src_stride0);
1551
1552    let src_col1 = src_base.offset(src_stride1);
1553    let s01 = *src_col1;
1554    let s11 = *src_col1.offset(src_stride0);
1555    let s21 = *src_col1.offset(2 * src_stride0);
1556    let s31 = *src_col1.offset(3 * src_stride0);
1557
1558    let src_col2 = src_base.offset(2 * src_stride1);
1559    let s02 = *src_col2;
1560    let s12 = *src_col2.offset(src_stride0);
1561    let s22 = *src_col2.offset(2 * src_stride0);
1562    let s32 = *src_col2.offset(3 * src_stride0);
1563
1564    let src_col3 = src_base.offset(3 * src_stride1);
1565    let s03 = *src_col3;
1566    let s13 = *src_col3.offset(src_stride0);
1567    let s23 = *src_col3.offset(2 * src_stride0);
1568    let s33 = *src_col3.offset(3 * src_stride0);
1569
1570    let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1571    *dst_row0 = scale * s00;
1572    *dst_row0.offset(dst_stride0) = scale * s01;
1573    *dst_row0.offset(2 * dst_stride0) = scale * s02;
1574    *dst_row0.offset(3 * dst_stride0) = scale * s03;
1575
1576    let dst_row1 = dst_row0.offset(dst_stride1);
1577    *dst_row1 = scale * s10;
1578    *dst_row1.offset(dst_stride0) = scale * s11;
1579    *dst_row1.offset(2 * dst_stride0) = scale * s12;
1580    *dst_row1.offset(3 * dst_stride0) = scale * s13;
1581
1582    let dst_row2 = dst_row0.offset(2 * dst_stride1);
1583    *dst_row2 = scale * s20;
1584    *dst_row2.offset(dst_stride0) = scale * s21;
1585    *dst_row2.offset(2 * dst_stride0) = scale * s22;
1586    *dst_row2.offset(3 * dst_stride0) = scale * s23;
1587
1588    let dst_row3 = dst_row0.offset(3 * dst_stride1);
1589    *dst_row3 = scale * s30;
1590    *dst_row3.offset(dst_stride0) = scale * s31;
1591    *dst_row3.offset(2 * dst_stride0) = scale * s32;
1592    *dst_row3.offset(3 * dst_stride0) = scale * s33;
1593}
1594
1595#[inline]
1596#[allow(clippy::too_many_arguments)]
1597unsafe fn copy_transpose_scale_2d_identity_tiled_raw<T>(
1598    dst: *mut T,
1599    dst_stride0: isize,
1600    dst_stride1: isize,
1601    src: *const T,
1602    src_stride0: isize,
1603    src_stride1: isize,
1604    src_rows: usize,
1605    src_cols: usize,
1606    scale: T,
1607    exec: Exec<'_>,
1608) where
1609    T: Copy + Mul<Output = T> + Send + Sync,
1610{
1611    const TILE: usize = 4;
1612    let row_full = src_rows / TILE * TILE;
1613    let col_full = src_cols / TILE * TILE;
1614
1615    let total = src_rows.saturating_mul(src_cols);
1616    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1617    if !exec.is_serial() {
1618        let dst_send = SendPtr(dst);
1619        let src_send = SendPtr(src as *mut T);
1620        let row_tiles = row_full / TILE;
1621        crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1622            for tile_i in tiles {
1623                let i = tile_i * TILE;
1624                let dst = dst_send.as_ptr();
1625                let src = src_send.as_const();
1626                unsafe {
1627                    let mut j = 0;
1628                    while j < col_full {
1629                        transpose_scale_4x4_identity(
1630                            dst,
1631                            dst_stride0,
1632                            dst_stride1,
1633                            src,
1634                            src_stride0,
1635                            src_stride1,
1636                            i,
1637                            j,
1638                            scale,
1639                        );
1640                        j += TILE;
1641                    }
1642                    for j in col_full..src_cols {
1643                        for ii in i..i + TILE {
1644                            *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1645                                scale
1646                                    * *src.offset(
1647                                        ii as isize * src_stride0 + j as isize * src_stride1,
1648                                    );
1649                        }
1650                    }
1651                }
1652            }
1653        });
1654        for i in row_full..src_rows {
1655            for j in 0..src_cols {
1656                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1657                    scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1658            }
1659        }
1660        return;
1661    }
1662
1663    let mut i = 0;
1664    while i < row_full {
1665        let mut j = 0;
1666        while j < col_full {
1667            transpose_scale_4x4_identity(
1668                dst,
1669                dst_stride0,
1670                dst_stride1,
1671                src,
1672                src_stride0,
1673                src_stride1,
1674                i,
1675                j,
1676                scale,
1677            );
1678            j += TILE;
1679        }
1680        for j in col_full..src_cols {
1681            for ii in i..i + TILE {
1682                *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1683                    scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1684            }
1685        }
1686        i += TILE;
1687    }
1688    for i in row_full..src_rows {
1689        for j in 0..src_cols {
1690            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1691                scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1692        }
1693    }
1694}
1695
1696#[inline]
1697unsafe fn try_copy_transpose_scale_2d_identity_tiled<T>(
1698    dest: &mut StridedViewMut<T>,
1699    src: &StridedView<T>,
1700    scale: T,
1701    exec: Exec<'_>,
1702) -> bool
1703where
1704    T: Copy + Mul<Output = T> + Send + Sync,
1705{
1706    if src.ndim() != 2 || dest.ndim() != 2 {
1707        return false;
1708    }
1709    let src_dims = src.dims();
1710    if dest.dims() != [src_dims[1], src_dims[0]] {
1711        return false;
1712    }
1713    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1714        return false;
1715    }
1716
1717    copy_transpose_scale_2d_identity_tiled_raw(
1718        dest.as_mut_ptr(),
1719        dest.strides()[0],
1720        dest.strides()[1],
1721        src.ptr(),
1722        src.strides()[0],
1723        src.strides()[1],
1724        src_dims[0],
1725        src_dims[1],
1726        scale,
1727        exec,
1728    );
1729    true
1730}
1731
1732#[inline]
1733#[allow(clippy::too_many_arguments)]
1734unsafe fn copy_transpose_scale_2d_loop<T>(
1735    dst: *mut T,
1736    dst_stride0: isize,
1737    dst_stride1: isize,
1738    src: *const T,
1739    src_stride0: isize,
1740    src_stride1: isize,
1741    src_rows: usize,
1742    src_cols: usize,
1743    scale: T,
1744    exec: Exec<'_>,
1745) where
1746    T: Copy + ElementOpApply + Mul<Output = T> + Send + Sync,
1747{
1748    let total = src_rows.saturating_mul(src_cols);
1749    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1750    if !exec.is_serial() {
1751        let dst_send = SendPtr(dst);
1752        let src_send = SendPtr(src as *mut T);
1753        if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1754            crate::threading::parallel_for_each(0..src_rows, exec, &|rows| {
1755                for i in rows {
1756                    let dst = dst_send.as_ptr();
1757                    let src = src_send.as_const();
1758                    unsafe {
1759                        for j in 0..src_cols {
1760                            let value = (*src
1761                                .offset(i as isize * src_stride0 + j as isize * src_stride1))
1762                            .transpose();
1763                            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1764                                scale * value;
1765                        }
1766                    }
1767                }
1768            });
1769        } else {
1770            crate::threading::parallel_for_each(0..src_cols, exec, &|columns| {
1771                for j in columns {
1772                    let dst = dst_send.as_ptr();
1773                    let src = src_send.as_const();
1774                    unsafe {
1775                        for i in 0..src_rows {
1776                            let value = (*src
1777                                .offset(i as isize * src_stride0 + j as isize * src_stride1))
1778                            .transpose();
1779                            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1780                                scale * value;
1781                        }
1782                    }
1783                }
1784            });
1785        }
1786        return;
1787    }
1788
1789    if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1790        for i in 0..src_rows {
1791            for j in 0..src_cols {
1792                let value =
1793                    (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1794                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1795            }
1796        }
1797    } else {
1798        for j in 0..src_cols {
1799            for i in 0..src_rows {
1800                let value =
1801                    (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1802                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1803            }
1804        }
1805    }
1806}
1807
1808/// Copy with transpose and scaling: `dest[j,i] = scale * src[i,j]`.
1809///
1810/// # Errors
1811///
1812/// Returns [`StridedError::RankMismatch`] when `src` or `dest` is not rank
1813/// 2, [`StridedError::ShapeMismatch`] when `dest` does not have the
1814/// transposed extents of `src`, and
1815/// [`StridedError::OffsetOverflow`] when the element count does not fit in
1816/// `usize`. When `scale` is one and `T`'s transpose is the identity, the
1817/// permutation path also returns [`StridedError::OffsetOverflow`] when the
1818/// layout cannot be expressed in `isize` stride arithmetic or a fused extent
1819/// overflows, and [`StridedError::NonInjectiveOutputLayout`] when the
1820/// destination layout cannot be proven representable and injective.
1821pub fn copy_transpose_scale_into<T>(
1822    dest: &mut StridedViewMut<T>,
1823    src: &StridedView<T>,
1824    scale: T,
1825    exec: Exec<'_>,
1826) -> Result<()>
1827where
1828    T: Copy + ElementOpApply + Mul<Output = T> + Zero + One + PartialEq + Send + Sync + 'static,
1829{
1830    let mut exec = exec;
1831    if src.ndim() != 2 || dest.ndim() != 2 {
1832        return Err(StridedError::RankMismatch(src.ndim(), 2));
1833    }
1834
1835    let src_dims = src.dims();
1836    let expected_dims = [src_dims[1], src_dims[0]];
1837    ensure_same_shape(dest.dims(), &expected_dims)?;
1838    // Reject an element count beyond usize (a huge stride-0 broadcast) up
1839    // front instead of looping over it.
1840    total_len(src_dims)?;
1841
1842    if scale == T::zero() {
1843        unsafe {
1844            if same_contiguous_layout(dest.dims(), &[dest.strides()]).is_some() {
1845                fill_contiguous(dest.as_mut_ptr(), total_len(dest.dims())?, T::zero());
1846            } else {
1847                fill_2d(
1848                    dest.as_mut_ptr(),
1849                    dest.dims()[0],
1850                    dest.dims()[1],
1851                    dest.strides()[0],
1852                    dest.strides()[1],
1853                    T::zero(),
1854                    exec,
1855                );
1856            }
1857        }
1858        return Ok(());
1859    }
1860
1861    let transpose_is_identity = element_transpose_is_identity::<T>();
1862
1863    unsafe {
1864        if transpose_is_identity
1865            && try_copy_transpose_scale_2d_f64_tiled_typed(dest, src, scale, exec.reborrow())
1866        {
1867            return Ok(());
1868        }
1869        if transpose_is_identity
1870            && try_copy_transpose_scale_2d_identity_tiled(dest, src, scale, exec.reborrow())
1871        {
1872            return Ok(());
1873        }
1874    }
1875
1876    if scale == T::one() && transpose_is_identity {
1877        let src_t = src.permute(&[1, 0])?;
1878        return crate::threading::copy_permuted(dest, &src_t, exec);
1879    }
1880
1881    unsafe {
1882        copy_transpose_scale_2d_loop(
1883            dest.as_mut_ptr(),
1884            dest.strides()[0],
1885            dest.strides()[1],
1886            src.ptr(),
1887            src.strides()[0],
1888            src.strides()[1],
1889            src_dims[0],
1890            src_dims[1],
1891            scale,
1892            exec,
1893        );
1894    }
1895    Ok(())
1896}
1897
1898#[cfg(test)]
1899#[path = "ops_view/tests/tiled_tests.rs"]
1900mod tiled_tests;