Skip to main content

strided_basic/
ops_view.rs

1//! High-level operations on dynamic-rank strided views.
2
3use crate::kernel::{
4    ensure_same_shape, for_each_inner_block_preordered, pointwise_plan, same_contiguous_layout,
5    sequential_contiguous_layout, total_len,
6};
7use crate::map_view::{copy_map_into, map_into, zip_map2_into};
8use crate::reduce_view::reduce;
9use crate::simd;
10use crate::view::{StridedView, StridedViewMut};
11use crate::{Exec, Result, StridedError};
12use num_traits::{One, Zero};
13use std::mem::MaybeUninit;
14use std::ops::{Add, Mul};
15use strided_view::{ElementOp, ElementOpApply};
16
17use crate::cost::KernelClass;
18use crate::fuse::compute_costs;
19use crate::threading::{
20    for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded, SendPtr,
21};
22// ============================================================================
23// Stride-specialized inner loop helpers for ops_view
24//
25// When all inner strides are 1 (contiguous in the innermost dimension),
26// we use slice-based iteration so LLVM can auto-vectorize effectively.
27// This mirrors the inner_loop_map* helpers in map_view.rs.
28// ============================================================================
29
30// INVARIANT: validated layouts keep all dereferences in bounds. Wrapping
31// advances permit an unused final cursor outside a reversed/gapped allocation.
32/// Inner loop for `add_acc`: `dst[i] += Op::apply(src[i])`.
33#[inline(always)]
34unsafe fn inner_loop_add<D: Copy + Add<S, Output = D>, S: Copy, Op: ElementOp<S>>(
35    dp: *mut D,
36    ds: isize,
37    sp: *const S,
38    ss: isize,
39    len: usize,
40) {
41    if ds == 1 && ss == 1 {
42        let dst = std::slice::from_raw_parts_mut(dp, len);
43        let src = std::slice::from_raw_parts(sp, len);
44        simd::dispatch_if_large(len, || {
45            for i in 0..len {
46                dst[i] = dst[i] + Op::apply(src[i]);
47            }
48        });
49    } else {
50        let mut dp = dp;
51        let mut sp = sp;
52        for _ in 0..len {
53            *dp = *dp + Op::apply(*sp);
54            dp = dp.wrapping_offset(ds);
55            sp = sp.wrapping_offset(ss);
56        }
57    }
58}
59
60/// Inner loop for `mul_acc`: `dst[i] *= Op::apply(src[i])`.
61#[inline(always)]
62unsafe fn inner_loop_mul<D: Copy + Mul<S, Output = D>, S: Copy, Op: ElementOp<S>>(
63    dp: *mut D,
64    ds: isize,
65    sp: *const S,
66    ss: isize,
67    len: usize,
68) {
69    if ds == 1 && ss == 1 {
70        let dst = std::slice::from_raw_parts_mut(dp, len);
71        let src = std::slice::from_raw_parts(sp, len);
72        simd::dispatch_if_large(len, || {
73            for i in 0..len {
74                dst[i] = dst[i] * Op::apply(src[i]);
75            }
76        });
77    } else {
78        let mut dp = dp;
79        let mut sp = sp;
80        for _ in 0..len {
81            *dp = *dp * Op::apply(*sp);
82            dp = dp.wrapping_offset(ds);
83            sp = sp.wrapping_offset(ss);
84        }
85    }
86}
87
88/// Inner loop for `axpy_acc`: `dst[i] = alpha * Op::apply(src[i]) + dst[i]`.
89#[inline(always)]
90unsafe fn inner_loop_axpy<
91    D: Copy + Add<D, Output = D>,
92    S: Copy,
93    A: Copy + Mul<S, Output = D>,
94    Op: ElementOp<S>,
95>(
96    dp: *mut D,
97    ds: isize,
98    sp: *const S,
99    ss: isize,
100    len: usize,
101    alpha: A,
102) {
103    if ds == 1 && ss == 1 {
104        let dst = std::slice::from_raw_parts_mut(dp, len);
105        let src = std::slice::from_raw_parts(sp, len);
106        simd::dispatch_if_large(len, || {
107            for i in 0..len {
108                dst[i] = alpha * Op::apply(src[i]) + dst[i];
109            }
110        });
111    } else {
112        let mut dp = dp;
113        let mut sp = sp;
114        for _ in 0..len {
115            *dp = alpha * Op::apply(*sp) + *dp;
116            dp = dp.wrapping_offset(ds);
117            sp = sp.wrapping_offset(ss);
118        }
119    }
120}
121
122/// Inner loop for `fma_acc`: `dst[i] += OpA::apply(a[i]) * OpB::apply(b[i])`.
123#[inline(always)]
124unsafe fn inner_loop_fma<
125    D: Copy + Add<D, Output = D>,
126    A: Copy + Mul<B, Output = D>,
127    B: Copy,
128    OpA: ElementOp<A>,
129    OpB: ElementOp<B>,
130>(
131    dp: *mut D,
132    ds: isize,
133    ap: *const A,
134    a_s: isize,
135    bp: *const B,
136    b_s: isize,
137    len: usize,
138) {
139    if ds == 1 && a_s == 1 && b_s == 1 {
140        let dst = std::slice::from_raw_parts_mut(dp, len);
141        let sa = std::slice::from_raw_parts(ap, len);
142        let sb = std::slice::from_raw_parts(bp, len);
143        simd::dispatch_if_large(len, || {
144            for i in 0..len {
145                dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
146            }
147        });
148    } else {
149        let mut dp = dp;
150        let mut ap = ap;
151        let mut bp = bp;
152        for _ in 0..len {
153            *dp = *dp + OpA::apply(*ap) * OpB::apply(*bp);
154            dp = dp.wrapping_offset(ds);
155            ap = ap.wrapping_offset(a_s);
156            bp = bp.wrapping_offset(b_s);
157        }
158    }
159}
160
161/// Inner loop for dot: `acc += OpA::apply(a[i]) * OpB::apply(b[i])`.
162#[inline(always)]
163unsafe fn inner_loop_dot<
164    A: Copy + Mul<B, Output = R>,
165    B: Copy,
166    R: Copy + Add<R, Output = R>,
167    OpA: ElementOp<A>,
168    OpB: ElementOp<B>,
169>(
170    ap: *const A,
171    a_s: isize,
172    bp: *const B,
173    b_s: isize,
174    len: usize,
175    mut acc: R,
176) -> R {
177    if a_s == 1 && b_s == 1 {
178        let sa = std::slice::from_raw_parts(ap, len);
179        let sb = std::slice::from_raw_parts(bp, len);
180        simd::dispatch_if_large(len, || {
181            for i in 0..len {
182                acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
183            }
184        });
185    } else {
186        let mut ap = ap;
187        let mut bp = bp;
188        for _ in 0..len {
189            acc = acc + OpA::apply(*ap) * OpB::apply(*bp);
190            ap = ap.wrapping_offset(a_s);
191            bp = bp.wrapping_offset(b_s);
192        }
193    }
194    acc
195}
196
197/// Copy elements from source to destination: `dest[i] = src[i]`.
198///
199/// # Errors
200///
201/// Returns [`StridedError::RankMismatch`] or
202/// [`StridedError::ShapeMismatch`] when the views describe different
203/// extents, and [`StridedError::OffsetOverflow`] when the element count of
204/// the shape does not fit in `usize`. The generic element path also returns
205/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
206/// cannot be proven representable and injective: two logical indices name
207/// one offset, or the address span is too large to represent. Nothing is
208/// written unless validation succeeds.
209pub fn copy_into<T: Copy + Send + Sync, Op: ElementOp<T>>(
210    dest: &mut StridedViewMut<T>,
211    src: &StridedView<T, Op>,
212    exec: Exec<'_>,
213) -> Result<()> {
214    ensure_same_shape(dest.dims(), src.dims())?;
215
216    let dst_ptr = dest.as_mut_ptr();
217    let src_ptr = src.ptr();
218    let dst_dims = dest.dims();
219    let dst_strides = dest.strides();
220    let src_strides = src.strides();
221
222    if sequential_contiguous_layout(
223        KernelClass::Copy,
224        dst_dims,
225        &[dst_strides, src_strides],
226        &exec,
227    )?
228    .is_some()
229    {
230        let len = total_len(dst_dims)?;
231        if Op::IS_IDENTITY {
232            debug_assert!(
233                {
234                    let nbytes = len
235                        .checked_mul(std::mem::size_of::<T>())
236                        .expect("copy size must not overflow");
237                    let dst_start = dst_ptr as usize;
238                    let src_start = src_ptr as usize;
239                    let dst_end = dst_start.saturating_add(nbytes);
240                    let src_end = src_start.saturating_add(nbytes);
241                    dst_end <= src_start || src_end <= dst_start
242                },
243                "overlapping src/dest is not supported"
244            );
245            unsafe { std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len) };
246        } else {
247            let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
248            let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
249            simd::dispatch_if_large(len, || {
250                for i in 0..len {
251                    dst[i] = Op::apply(src[i]);
252                }
253            });
254        }
255        return Ok(());
256    }
257
258    copy_map_into(dest, src, |x| x, exec)
259}
260
261/// Copy initialized values into a potentially uninitialized destination.
262///
263/// On success every logical destination element is initialized; holes are
264/// untouched. Accepts identity views; use `map_into` for element operations
265/// such as conjugation. Equal contiguous f32/f64 layouts and every other type
266/// keep the element map; a non-contiguous f32/f64 copy uses the permutation
267/// engine, split within the budget of `exec`.
268///
269/// # Errors
270/// Returns a shape/rank error for mismatched views, `NonInjectiveOutputLayout`
271/// for overlapping destination elements, or `OffsetOverflow` for size overflow.
272///
273/// # Examples
274/// ```
275/// use std::mem::MaybeUninit;
276/// use strided_basic::{copy_into_uninit, Exec, StridedView, StridedViewMut};
277/// let src = [1.0_f64, 2.0];
278/// let mut dst = [MaybeUninit::uninit(); 2];
279/// copy_into_uninit(
280///     &mut StridedViewMut::new(&mut dst, &[2], &[1], 0).unwrap(),
281///     &StridedView::new(&src, &[2], &[1], 0).unwrap(),
282///     Exec::serial(),
283/// ).unwrap();
284/// // SAFETY: the successful copy initialized both destination elements.
285/// assert_eq!(unsafe { dst[1].assume_init() }, 2.0);
286/// ```
287pub fn copy_into_uninit<T: Copy + Send + Sync + 'static>(
288    dest: &mut StridedViewMut<MaybeUninit<T>>,
289    src: &StridedView<T>,
290    exec: Exec<'_>,
291) -> Result<()> {
292    ensure_same_shape(dest.dims(), src.dims())?;
293    if !crate::layout_check::is_injective_layout(dest.dims(), dest.strides()) {
294        return Err(StridedError::NonInjectiveOutputLayout);
295    }
296    if src.is_empty() {
297        return Ok(());
298    }
299    let _len = src
300        .dims()
301        .iter()
302        .try_fold(1usize, |n, &dim| n.checked_mul(dim))
303        .ok_or(StridedError::OffsetOverflow)?;
304    // The permutation engine's 4/8-byte paths reinterpret storage as native
305    // floats. Restrict that path to those exact types: arbitrary Copy types can
306    // contain uninitialized padding or have weaker alignment (e.g. Complex32).
307    let native_float = std::any::TypeId::of::<T>() == std::any::TypeId::of::<f32>()
308        || std::any::TypeId::of::<T>() == std::any::TypeId::of::<f64>();
309    // Equal contiguous layouts go to the element map, whose contiguous path
310    // parallelizes a large copy; the permutation engine has only a serial
311    // contiguous shortcut. Non-contiguous native-float copies use the
312    // permutation engine, serial or parallel.
313    if !native_float
314        || same_contiguous_layout(dest.dims(), &[dest.strides(), src.strides()]).is_some()
315    {
316        return copy_map_into(dest, src, MaybeUninit::new, exec);
317    }
318    // INVARIANT: the permutation engine selects its float paths by the
319    // source element type and only stores into the destination, so an
320    // uninitialized destination with an initialized source is valid.
321    crate::threading::copy_permuted_uninit(dest, src, exec)
322}
323
324/// Copy elements from `src` to `dst`, optimized for col-major destination.
325///
326/// Delegates to the centralized permutation-copy dispatch for the actual work,
327/// inside the budget of `exec`: the same explicit-token contract as
328/// [`copy_into`], so a large copy splits across the token's slots instead of
329/// always staying on the calling thread (strided-rs#212).
330///
331/// # Errors
332///
333/// The rejection set of that dispatch: [`StridedError::RankMismatch`] or
334/// [`StridedError::ShapeMismatch`] when the views describe different
335/// extents, [`StridedError::OffsetOverflow`] when the layout cannot be
336/// expressed in `isize` stride arithmetic or a fused extent overflows, and
337/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
338/// cannot be proven representable and injective. Nothing is written unless
339/// validation succeeds.
340pub fn copy_into_col_major<T: Copy + Send + Sync + 'static>(
341    dst: &mut StridedViewMut<T>,
342    src: &StridedView<T>,
343    exec: Exec<'_>,
344) -> Result<()> {
345    strided_perm::copy_into_col_major(dst, src, exec)
346}
347
348/// Accumulating element-wise addition: `dest[i] += src[i]`.
349///
350/// Source may have a different element type from destination.
351///
352/// # Errors
353///
354/// Returns [`StridedError::RankMismatch`] or
355/// [`StridedError::ShapeMismatch`] when the views describe different
356/// extents, and [`StridedError::OffsetOverflow`] when the element count of
357/// the shape does not fit in `usize`. Every check runs before the first
358/// write, so a rejected call leaves `dest` untouched.
359pub fn add_acc<
360    D: Copy + Add<S, Output = D> + Send + Sync,
361    S: Copy + Send + Sync,
362    Op: ElementOp<S>,
363>(
364    dest: &mut StridedViewMut<D>,
365    src: &StridedView<S, Op>,
366    exec: Exec<'_>,
367) -> Result<()> {
368    ensure_same_shape(dest.dims(), src.dims())?;
369
370    let dst_ptr = dest.as_mut_ptr();
371    let src_ptr = src.ptr();
372    let dst_dims = dest.dims();
373    let dst_strides = dest.strides();
374    let src_strides = src.strides();
375
376    if sequential_contiguous_layout(
377        KernelClass::Map,
378        dst_dims,
379        &[dst_strides, src_strides],
380        &exec,
381    )?
382    .is_some()
383    {
384        let len = total_len(dst_dims)?;
385        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
386        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
387        simd::dispatch_if_large(len, || {
388            for i in 0..len {
389                dst[i] = dst[i] + Op::apply(src[i]);
390            }
391        });
392        return Ok(());
393    }
394
395    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
396    let total = total_len(dst_dims)?;
397    let (fused_dims, ordered_strides, plan) = pointwise_plan(
398        dst_dims,
399        &strides_list,
400        Some(0),
401        &[std::mem::size_of::<D>(), std::mem::size_of::<S>()],
402        total,
403    );
404
405    let total = total_len(&fused_dims)?;
406    if is_parallel(KernelClass::Map, total, &exec) {
407        let dst_send = SendPtr(dst_ptr);
408        let src_send = SendPtr(src_ptr as *mut S);
409
410        let costs = compute_costs(&ordered_strides);
411        let initial_offsets: crate::inline::OperandVec<isize> =
412            strides_list.iter().map(|_| 0isize).collect();
413        return mapreduce_threaded(
414            &fused_dims,
415            &plan.block,
416            &ordered_strides,
417            &initial_offsets,
418            &costs,
419            KernelClass::Map,
420            exec,
421            &|dims, blocks, strides_list, offsets| {
422                for_each_inner_block_with_offsets(
423                    dims,
424                    blocks,
425                    strides_list,
426                    offsets,
427                    |offsets, len, strides| {
428                        unsafe {
429                            inner_loop_add::<D, S, Op>(
430                                dst_send.as_ptr().offset(offsets[0]),
431                                strides[0],
432                                src_send.as_const().offset(offsets[1]),
433                                strides[1],
434                                len,
435                            )
436                        };
437                        Ok(())
438                    },
439                )
440            },
441        );
442    }
443
444    let initial_offsets: crate::inline::OperandVec<isize> =
445        ordered_strides.iter().map(|_| 0isize).collect();
446    for_each_inner_block_preordered(
447        &fused_dims,
448        &plan.block,
449        &ordered_strides,
450        &initial_offsets,
451        |offsets, len, strides| {
452            unsafe {
453                inner_loop_add::<D, S, Op>(
454                    dst_ptr.offset(offsets[0]),
455                    strides[0],
456                    src_ptr.offset(offsets[1]),
457                    strides[1],
458                    len,
459                )
460            };
461            Ok(())
462        },
463    )
464}
465
466/// Accumulating element-wise multiplication: `dest[i] *= src[i]`.
467///
468/// Source may have a different element type from destination.
469///
470/// # Errors
471///
472/// Returns [`StridedError::RankMismatch`] or
473/// [`StridedError::ShapeMismatch`] when the views describe different
474/// extents, and [`StridedError::OffsetOverflow`] when the element count of
475/// the shape does not fit in `usize`. Every check runs before the first
476/// write, so a rejected call leaves `dest` untouched.
477pub fn mul_acc<
478    D: Copy + Mul<S, Output = D> + Send + Sync,
479    S: Copy + Send + Sync,
480    Op: ElementOp<S>,
481>(
482    dest: &mut StridedViewMut<D>,
483    src: &StridedView<S, Op>,
484    exec: Exec<'_>,
485) -> Result<()> {
486    ensure_same_shape(dest.dims(), src.dims())?;
487
488    let dst_ptr = dest.as_mut_ptr();
489    let src_ptr = src.ptr();
490    let dst_dims = dest.dims();
491    let dst_strides = dest.strides();
492    let src_strides = src.strides();
493
494    if sequential_contiguous_layout(
495        KernelClass::Map,
496        dst_dims,
497        &[dst_strides, src_strides],
498        &exec,
499    )?
500    .is_some()
501    {
502        let len = total_len(dst_dims)?;
503        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
504        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
505        simd::dispatch_if_large(len, || {
506            for i in 0..len {
507                dst[i] = dst[i] * Op::apply(src[i]);
508            }
509        });
510        return Ok(());
511    }
512
513    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
514    let total = total_len(dst_dims)?;
515    let (fused_dims, ordered_strides, plan) = pointwise_plan(
516        dst_dims,
517        &strides_list,
518        Some(0),
519        &[std::mem::size_of::<D>(), std::mem::size_of::<S>()],
520        total,
521    );
522
523    let total = total_len(&fused_dims)?;
524    if is_parallel(KernelClass::Map, total, &exec) {
525        let dst_send = SendPtr(dst_ptr);
526        let src_send = SendPtr(src_ptr as *mut S);
527
528        let costs = compute_costs(&ordered_strides);
529        let initial_offsets: crate::inline::OperandVec<isize> =
530            strides_list.iter().map(|_| 0isize).collect();
531        return mapreduce_threaded(
532            &fused_dims,
533            &plan.block,
534            &ordered_strides,
535            &initial_offsets,
536            &costs,
537            KernelClass::Map,
538            exec,
539            &|dims, blocks, strides_list, offsets| {
540                for_each_inner_block_with_offsets(
541                    dims,
542                    blocks,
543                    strides_list,
544                    offsets,
545                    |offsets, len, strides| {
546                        unsafe {
547                            inner_loop_mul::<D, S, Op>(
548                                dst_send.as_ptr().offset(offsets[0]),
549                                strides[0],
550                                src_send.as_const().offset(offsets[1]),
551                                strides[1],
552                                len,
553                            )
554                        };
555                        Ok(())
556                    },
557                )
558            },
559        );
560    }
561
562    let initial_offsets: crate::inline::OperandVec<isize> =
563        ordered_strides.iter().map(|_| 0isize).collect();
564    for_each_inner_block_preordered(
565        &fused_dims,
566        &plan.block,
567        &ordered_strides,
568        &initial_offsets,
569        |offsets, len, strides| {
570            unsafe {
571                inner_loop_mul::<D, S, Op>(
572                    dst_ptr.offset(offsets[0]),
573                    strides[0],
574                    src_ptr.offset(offsets[1]),
575                    strides[1],
576                    len,
577                )
578            };
579            Ok(())
580        },
581    )
582}
583
584/// Accumulating AXPY: `dest[i] = alpha * src[i] + dest[i]`.
585///
586/// Alpha, source, and destination may have different element types.
587///
588/// # Errors
589///
590/// Returns [`StridedError::RankMismatch`] or
591/// [`StridedError::ShapeMismatch`] when the views describe different
592/// extents, and [`StridedError::OffsetOverflow`] when the element count of
593/// the shape does not fit in `usize`. Every check runs before the first
594/// write, so a rejected call leaves `dest` untouched.
595pub fn axpy_acc<D, S, A, Op>(
596    dest: &mut StridedViewMut<D>,
597    src: &StridedView<S, Op>,
598    alpha: A,
599    exec: Exec<'_>,
600) -> Result<()>
601where
602    A: Copy + Mul<S, Output = D> + Sync,
603    D: Copy + Add<D, Output = D> + Send + Sync,
604    S: Copy + Send + Sync,
605    Op: ElementOp<S>,
606{
607    ensure_same_shape(dest.dims(), src.dims())?;
608
609    let dst_ptr = dest.as_mut_ptr();
610    let src_ptr = src.ptr();
611    let dst_dims = dest.dims();
612    let dst_strides = dest.strides();
613    let src_strides = src.strides();
614
615    if sequential_contiguous_layout(
616        KernelClass::Map,
617        dst_dims,
618        &[dst_strides, src_strides],
619        &exec,
620    )?
621    .is_some()
622    {
623        let len = total_len(dst_dims)?;
624        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
625        let src = unsafe { std::slice::from_raw_parts(src_ptr, len) };
626        simd::dispatch_if_large(len, || {
627            for i in 0..len {
628                dst[i] = alpha * Op::apply(src[i]) + dst[i];
629            }
630        });
631        return Ok(());
632    }
633
634    let strides_list: [&[isize]; 2] = [dst_strides, src_strides];
635    let total = total_len(dst_dims)?;
636    let (fused_dims, ordered_strides, plan) = pointwise_plan(
637        dst_dims,
638        &strides_list,
639        Some(0),
640        &[std::mem::size_of::<D>(), std::mem::size_of::<S>()],
641        total,
642    );
643
644    let total = total_len(&fused_dims)?;
645    if is_parallel(KernelClass::Map, total, &exec) {
646        let dst_send = SendPtr(dst_ptr);
647        let src_send = SendPtr(src_ptr as *mut S);
648
649        let costs = compute_costs(&ordered_strides);
650        let initial_offsets: crate::inline::OperandVec<isize> =
651            strides_list.iter().map(|_| 0isize).collect();
652        return mapreduce_threaded(
653            &fused_dims,
654            &plan.block,
655            &ordered_strides,
656            &initial_offsets,
657            &costs,
658            KernelClass::Map,
659            exec,
660            &|dims, blocks, strides_list, offsets| {
661                for_each_inner_block_with_offsets(
662                    dims,
663                    blocks,
664                    strides_list,
665                    offsets,
666                    |offsets, len, strides| {
667                        unsafe {
668                            inner_loop_axpy::<D, S, A, Op>(
669                                dst_send.as_ptr().offset(offsets[0]),
670                                strides[0],
671                                src_send.as_const().offset(offsets[1]),
672                                strides[1],
673                                len,
674                                alpha,
675                            )
676                        };
677                        Ok(())
678                    },
679                )
680            },
681        );
682    }
683
684    let initial_offsets: crate::inline::OperandVec<isize> =
685        ordered_strides.iter().map(|_| 0isize).collect();
686    for_each_inner_block_preordered(
687        &fused_dims,
688        &plan.block,
689        &ordered_strides,
690        &initial_offsets,
691        |offsets, len, strides| {
692            unsafe {
693                inner_loop_axpy::<D, S, A, Op>(
694                    dst_ptr.offset(offsets[0]),
695                    strides[0],
696                    src_ptr.offset(offsets[1]),
697                    strides[1],
698                    len,
699                    alpha,
700                )
701            };
702            Ok(())
703        },
704    )
705}
706
707/// Accumulating fused multiply-add: `dest[i] += OpA::apply(a[i]) * OpB::apply(b[i])`.
708///
709/// Operands may have different element types. Element operations are applied lazily.
710///
711/// # Errors
712///
713/// Returns [`StridedError::RankMismatch`] or
714/// [`StridedError::ShapeMismatch`] when `dest` and an operand describe
715/// different extents, and [`StridedError::OffsetOverflow`] when the element
716/// count of the shape does not fit in `usize`. Every check runs before the
717/// first write, so a rejected call leaves `dest` untouched.
718pub fn fma_acc<D, A, B, OpA, OpB>(
719    dest: &mut StridedViewMut<D>,
720    a: &StridedView<A, OpA>,
721    b: &StridedView<B, OpB>,
722    exec: Exec<'_>,
723) -> Result<()>
724where
725    A: Copy + Mul<B, Output = D> + Send + Sync,
726    B: Copy + Send + Sync,
727    D: Copy + Add<D, Output = D> + Send + Sync,
728    OpA: ElementOp<A>,
729    OpB: ElementOp<B>,
730{
731    ensure_same_shape(dest.dims(), a.dims())?;
732    ensure_same_shape(dest.dims(), b.dims())?;
733
734    let dst_ptr = dest.as_mut_ptr();
735    let a_ptr = a.ptr();
736    let b_ptr = b.ptr();
737    let dst_dims = dest.dims();
738    let dst_strides = dest.strides();
739    let a_strides = a.strides();
740    let b_strides = b.strides();
741
742    if sequential_contiguous_layout(
743        KernelClass::Map,
744        dst_dims,
745        &[dst_strides, a_strides, b_strides],
746        &exec,
747    )?
748    .is_some()
749    {
750        let len = total_len(dst_dims)?;
751        let dst = unsafe { std::slice::from_raw_parts_mut(dst_ptr, len) };
752        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
753        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
754        simd::dispatch_if_large(len, || {
755            for i in 0..len {
756                dst[i] = dst[i] + OpA::apply(sa[i]) * OpB::apply(sb[i]);
757            }
758        });
759        return Ok(());
760    }
761
762    let strides_list: [&[isize]; 3] = [dst_strides, a_strides, b_strides];
763    let total = total_len(dst_dims)?;
764    let (fused_dims, ordered_strides, plan) = pointwise_plan(
765        dst_dims,
766        &strides_list,
767        Some(0),
768        &[
769            std::mem::size_of::<D>(),
770            std::mem::size_of::<A>(),
771            std::mem::size_of::<B>(),
772        ],
773        total,
774    );
775
776    let total = total_len(&fused_dims)?;
777    if is_parallel(KernelClass::Map, total, &exec) {
778        let dst_send = SendPtr(dst_ptr);
779        let a_send = SendPtr(a_ptr as *mut A);
780        let b_send = SendPtr(b_ptr as *mut B);
781
782        let costs = compute_costs(&ordered_strides);
783        let initial_offsets: crate::inline::OperandVec<isize> =
784            strides_list.iter().map(|_| 0isize).collect();
785        return mapreduce_threaded(
786            &fused_dims,
787            &plan.block,
788            &ordered_strides,
789            &initial_offsets,
790            &costs,
791            KernelClass::Map,
792            exec,
793            &|dims, blocks, strides_list, offsets| {
794                for_each_inner_block_with_offsets(
795                    dims,
796                    blocks,
797                    strides_list,
798                    offsets,
799                    |offsets, len, strides| {
800                        unsafe {
801                            inner_loop_fma::<D, A, B, OpA, OpB>(
802                                dst_send.as_ptr().offset(offsets[0]),
803                                strides[0],
804                                a_send.as_const().offset(offsets[1]),
805                                strides[1],
806                                b_send.as_const().offset(offsets[2]),
807                                strides[2],
808                                len,
809                            )
810                        };
811                        Ok(())
812                    },
813                )
814            },
815        );
816    }
817
818    let initial_offsets: crate::inline::OperandVec<isize> =
819        ordered_strides.iter().map(|_| 0isize).collect();
820    for_each_inner_block_preordered(
821        &fused_dims,
822        &plan.block,
823        &ordered_strides,
824        &initial_offsets,
825        |offsets, len, strides| {
826            unsafe {
827                inner_loop_fma::<D, A, B, OpA, OpB>(
828                    dst_ptr.offset(offsets[0]),
829                    strides[0],
830                    a_ptr.offset(offsets[1]),
831                    strides[1],
832                    b_ptr.offset(offsets[2]),
833                    strides[2],
834                    len,
835                )
836            };
837            Ok(())
838        },
839    )
840}
841
842/// Sum all elements: `sum(src)`.
843///
844/// # Evaluation order
845///
846/// The result is independent of the pool when the call runs with a serial
847/// token ([`Exec::serial`]) or a budget of one. With a larger budget a
848/// contiguous identity operand above the threading threshold is summed per
849/// task and the partial sums are combined in a tree, so the result is
850/// repeatable for a fixed budget but can differ between budgets.
851///
852/// Neither path is a strict left to right sum: a contiguous run is folded with
853/// independent SIMD or scalar lanes (see [`reduce`] for the lane scheme of the
854/// generic path). The last bits can therefore differ from a naive loop, and no
855/// bitwise equality across machines or target features is guaranteed.
856///
857/// For a deterministic accumulation of `f32` data in `f64`, use
858/// [`zip_reduce`](crate::zip_reduce) with an `f64` map and a `0.0f64` init.
859///
860/// # Allocation
861///
862/// At rank ≤ [`INLINE_RANK`](strided_view::INLINE_RANK) the view metadata and
863/// the traversal plan are stored inline, so a call performs no heap allocation.
864///
865/// # Errors
866///
867/// Returns [`StridedError::OffsetOverflow`] when the element count of the
868/// view's shape does not fit in `usize`, the only failure this entry can
869/// produce. That check runs first, before any planning or read.
870pub fn sum<
871    T: Copy + Zero + Add<Output = T> + Send + Sync + simd::MaybeSimdOps,
872    Op: ElementOp<T>,
873>(
874    src: &StridedView<T, Op>,
875    exec: Exec<'_>,
876) -> Result<T> {
877    // A zero-element sum is the additive identity. Return before any planning
878    // so an empty view with an enormous stride never reaches block sizing.
879    if total_len(src.dims())? == 0 {
880        return Ok(T::zero());
881    }
882    // SIMD fast path: contiguous Identity view with SIMD support
883    if Op::IS_IDENTITY && same_contiguous_layout(src.dims(), &[src.strides()]).is_some() {
884        let len = total_len(src.dims())?;
885        let src_slice = unsafe { std::slice::from_raw_parts(src.ptr(), len) };
886
887        // The partial sums of the halves are combined in a tree; a type
888        // without SIMD support takes the generic reduction below.
889        if T::try_simd_sum(&[]).is_some() && is_parallel(KernelClass::Reduce, len, &exec) {
890            return Ok(crate::threading::parallel_map_reduce(
891                0..len,
892                crate::cost::for_len_tree(exec, KernelClass::Reduce, len),
893                &|range| T::try_simd_sum(&src_slice[range]).unwrap(),
894                &|left, right| left + right,
895            ));
896        }
897
898        if let Some(result) = T::try_simd_sum(src_slice) {
899            return Ok(result);
900        }
901    }
902    reduce(src, |x| x, |a, b| a + b, T::zero(), exec)
903}
904
905/// Dot product: `sum(OpA::apply(a[i]) * OpB::apply(b[i]))`.
906///
907/// Operands may have different element types. Result type `R` must be `A * B`.
908/// SIMD fast path fires only when `A == B == R` (same type) and both Identity ops.
909///
910/// A widening dot — `f32` operands with an `f64` accumulator — is not
911/// expressible here because `R` is pinned to `A * B`; use
912/// [`zip_reduce`](crate::zip_reduce) with a
913/// `|x, y| x as f64 * y as f64` map instead.
914///
915/// # Evaluation order
916///
917/// This entry takes no parallel branch and no token, so its result does not
918/// depend on any pool: it is computed on the calling thread. The homogeneous contiguous fast path folds with the SIMD
919/// lane scheme of the element type; every other layout folds a single
920/// accumulator along the traversal order of the fused plan, which is left to
921/// right over each unit-stride run. Neither is guaranteed to be bitwise equal
922/// across machines or target features.
923///
924/// # Allocation
925///
926/// At rank ≤ [`INLINE_RANK`](strided_view::INLINE_RANK) the view metadata and
927/// the traversal plan are stored inline, so a call performs no heap allocation.
928///
929/// # Errors
930///
931/// Returns [`StridedError::RankMismatch`] or
932/// [`StridedError::ShapeMismatch`] when the two operands describe different
933/// extents, and [`StridedError::OffsetOverflow`] when the element count of
934/// their shape does not fit in `usize`.
935pub fn dot<A, B, R, OpA, OpB>(a: &StridedView<A, OpA>, b: &StridedView<B, OpB>) -> Result<R>
936where
937    A: Copy + Mul<B, Output = R> + Send + Sync + 'static,
938    B: Copy + Send + Sync + 'static,
939    R: Copy + Zero + Add<Output = R> + Send + Sync + simd::MaybeSimdOps + 'static,
940    OpA: ElementOp<A>,
941    OpB: ElementOp<B>,
942{
943    ensure_same_shape(a.dims(), b.dims())?;
944
945    let a_ptr = a.ptr();
946    let b_ptr = b.ptr();
947    let a_strides = a.strides();
948    let b_strides = b.strides();
949    let a_dims = a.dims();
950    let len = total_len(a_dims)?;
951
952    // A zero-element dot product is the additive identity. Return before any
953    // planning so an empty view with an enormous stride never reaches block
954    // sizing.
955    if len == 0 {
956        return Ok(R::zero());
957    }
958
959    if same_contiguous_layout(a_dims, &[a_strides, b_strides]).is_some() {
960        // SIMD fast path: both contiguous, both Identity ops, same type
961        if OpA::IS_IDENTITY
962            && OpB::IS_IDENTITY
963            && std::any::TypeId::of::<A>() == std::any::TypeId::of::<R>()
964            && std::any::TypeId::of::<B>() == std::any::TypeId::of::<R>()
965        {
966            let sa = unsafe { std::slice::from_raw_parts(a_ptr as *const R, len) };
967            let sb = unsafe { std::slice::from_raw_parts(b_ptr as *const R, len) };
968            if let Some(result) = R::try_simd_dot(sa, sb) {
969                return Ok(result);
970            }
971        }
972
973        // Generic contiguous fast path
974        let sa = unsafe { std::slice::from_raw_parts(a_ptr, len) };
975        let sb = unsafe { std::slice::from_raw_parts(b_ptr, len) };
976        let mut acc = R::zero();
977        simd::dispatch_if_large(len, || {
978            for i in 0..len {
979                acc = acc + OpA::apply(sa[i]) * OpB::apply(sb[i]);
980            }
981        });
982        return Ok(acc);
983    }
984
985    let strides_list: [&[isize]; 2] = [a_strides, b_strides];
986    // The accumulator `R` is not a memory operand, so it has no element size.
987    let total = total_len(a_dims)?;
988    // A fold: its tiles order `acc`, so it keeps the model's (#287).
989    let (fused_dims, ordered_strides, plan) = crate::kernel::fold_plan(
990        a_dims,
991        &strides_list,
992        0,
993        &[std::mem::size_of::<A>(), std::mem::size_of::<B>()],
994        total,
995    );
996
997    let mut acc = R::zero();
998    let initial_offsets: crate::inline::OperandVec<isize> =
999        ordered_strides.iter().map(|_| 0isize).collect();
1000    for_each_inner_block_preordered(
1001        &fused_dims,
1002        &plan.block,
1003        &ordered_strides,
1004        &initial_offsets,
1005        |offsets, len, strides| {
1006            acc = unsafe {
1007                inner_loop_dot::<A, B, R, OpA, OpB>(
1008                    a_ptr.offset(offsets[0]),
1009                    strides[0],
1010                    b_ptr.offset(offsets[1]),
1011                    strides[1],
1012                    len,
1013                    acc,
1014                )
1015            };
1016            Ok(())
1017        },
1018    )?;
1019
1020    Ok(acc)
1021}
1022
1023/// Symmetrize a square matrix: `dest = (src + src^T) / 2`.
1024///
1025/// # Errors
1026///
1027/// Returns [`StridedError::RankMismatch`] when `src` is not rank 2,
1028/// [`StridedError::NonSquare`] when its two extents differ, and
1029/// [`StridedError::ScalarConversion`] only when `T::from_f64(0.5)` returns
1030/// `None`. `num_traits::FromPrimitive::from_f64` truncates for the primitive
1031/// integer types (`i32::from_f64(0.5) == Some(0)`), so an integer `T` is not
1032/// rejected: it scales by zero, and the result is zero rather than a half.
1033/// Writing through [`zip_map2_into`](crate::zip_map2_into) adds
1034/// [`StridedError::RankMismatch`] or [`StridedError::ShapeMismatch`] when
1035/// `dest` and the transposed `src` describe different extents, and
1036/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1037/// cannot be proven representable and injective: two logical indices name
1038/// one offset, or the element count or address span is too large to
1039/// represent. Nothing is written unless validation succeeds.
1040pub fn symmetrize_into<T>(
1041    dest: &mut StridedViewMut<T>,
1042    src: &StridedView<T>,
1043    exec: Exec<'_>,
1044) -> Result<()>
1045where
1046    T: Copy
1047        + Add<Output = T>
1048        + Mul<Output = T>
1049        + num_traits::FromPrimitive
1050        + std::ops::Div<Output = T>
1051        + Send
1052        + Sync,
1053{
1054    if src.ndim() != 2 {
1055        return Err(StridedError::RankMismatch(src.ndim(), 2));
1056    }
1057    let rows = src.dims()[0];
1058    let cols = src.dims()[1];
1059    if rows != cols {
1060        return Err(StridedError::NonSquare { rows, cols });
1061    }
1062
1063    let src_t = src.permute(&[1, 0])?;
1064    let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1065
1066    zip_map2_into(dest, src, &src_t, |a, b| (a + b) * half, exec)
1067}
1068
1069/// Conjugate-symmetrize a square matrix: `dest = (src + conj(src^T)) / 2`.
1070///
1071/// # Errors
1072///
1073/// Returns [`StridedError::RankMismatch`] when `src` is not rank 2,
1074/// [`StridedError::NonSquare`] when its two extents differ, and
1075/// [`StridedError::ScalarConversion`] only when `T::from_f64(0.5)` returns
1076/// `None`. `num_traits::FromPrimitive::from_f64` truncates for the primitive
1077/// integer types (`i32::from_f64(0.5) == Some(0)`), so an integer `T` is not
1078/// rejected: it scales by zero, and the result is zero rather than a half.
1079/// Writing through [`zip_map2_into`](crate::zip_map2_into) adds
1080/// [`StridedError::RankMismatch`] or [`StridedError::ShapeMismatch`] when
1081/// `dest` and the adjoint of `src` describe different extents, and
1082/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1083/// cannot be proven representable and injective: two logical indices name
1084/// one offset, or the element count or address span is too large to
1085/// represent. Nothing is written unless validation succeeds.
1086pub fn symmetrize_conj_into<T>(
1087    dest: &mut StridedViewMut<T>,
1088    src: &StridedView<T>,
1089    exec: Exec<'_>,
1090) -> Result<()>
1091where
1092    T: Copy
1093        + ElementOpApply
1094        + Add<Output = T>
1095        + Mul<Output = T>
1096        + num_traits::FromPrimitive
1097        + std::ops::Div<Output = T>
1098        + Send
1099        + Sync,
1100{
1101    if src.ndim() != 2 {
1102        return Err(StridedError::RankMismatch(src.ndim(), 2));
1103    }
1104    let rows = src.dims()[0];
1105    let cols = src.dims()[1];
1106    if rows != cols {
1107        return Err(StridedError::NonSquare { rows, cols });
1108    }
1109
1110    // adjoint = conj + transpose
1111    let src_adj = src.adjoint_2d()?;
1112    let half = T::from_f64(0.5).ok_or(StridedError::ScalarConversion)?;
1113
1114    zip_map2_into(dest, src, &src_adj, |a, b| (a + b) * half, exec)
1115}
1116
1117/// Overwriting copy with scaling: `dest[i] = scale * src[i]`.
1118///
1119/// Scale, source, and destination may have different element types.
1120///
1121/// # Errors
1122///
1123/// Returns [`StridedError::RankMismatch`] or
1124/// [`StridedError::ShapeMismatch`] when the views describe different
1125/// extents, and [`StridedError::NonInjectiveOutputLayout`] when the
1126/// destination layout cannot be proven representable and injective: two
1127/// logical indices name one offset, or the element count or address span is
1128/// too large to represent. Nothing is written unless validation succeeds.
1129pub fn copy_scale_into<D, S, A, Op>(
1130    dest: &mut StridedViewMut<D>,
1131    src: &StridedView<S, Op>,
1132    scale: A,
1133    exec: Exec<'_>,
1134) -> Result<()>
1135where
1136    A: Copy + Mul<S, Output = D> + Sync,
1137    D: Copy + Send + Sync,
1138    S: Copy + Send + Sync,
1139    Op: ElementOp<S>,
1140{
1141    map_into(dest, src, |x| scale * x, exec)
1142}
1143
1144/// Overwriting copy with complex conjugation: `dest[i] = conj(src[i])`.
1145///
1146/// A non-contiguous copy of a complex element type is handed to the permutation
1147/// engine's conjugating store ([`strided_perm::copy_conj_into`], HPTT's
1148/// `conjA`) when that engine is measured to beat the element map for the shape;
1149/// every other copy keeps the element map. The engine conjugates each element
1150/// as it stores it, so the operation costs no extra pass over memory. Which
1151/// route is taken is measured, not guessed: `benches/conj_route_467.rs`
1152/// reproduces the table and `docs/worklogs/2026-11-15-issue-467-conj-route.md`
1153/// records it, with the bound and the rows that fix it. A real element type's
1154/// `conj` is the identity, so it always keeps the element map; use
1155/// [`copy_into`] for those.
1156///
1157/// # Errors
1158///
1159/// The rejection set of [`copy_into`]: [`StridedError::RankMismatch`] or
1160/// [`StridedError::ShapeMismatch`] when the views describe different
1161/// extents, [`StridedError::OffsetOverflow`] when the element count of the
1162/// shape does not fit in `usize`, and
1163/// [`StridedError::NonInjectiveOutputLayout`] when the destination layout
1164/// cannot be proven representable and injective. Nothing is written unless
1165/// validation succeeds.
1166///
1167/// The engine route reports one of those differently: it plans the permutation
1168/// before it checks the destination for injectivity, so an unrepresentable
1169/// destination keeps reporting [`StridedError::OffsetOverflow`] (issue #263)
1170/// where [`copy_into`] reports [`StridedError::NonInjectiveOutputLayout`] under
1171/// its "cannot be proven representable and injective" case. Nothing is written
1172/// on either route.
1173pub fn copy_conj_into<T: ElementOpApply + Send + Sync + 'static>(
1174    dest: &mut StridedViewMut<T>,
1175    src: &StridedView<T>,
1176    exec: Exec<'_>,
1177) -> Result<()> {
1178    if conj_perm_engine_route::<T>(dest.dims(), dest.strides(), src.strides()) {
1179        return crate::threading::copy_conj_permuted(dest, src, exec);
1180    }
1181    let src_conj = src.conj();
1182    copy_into(dest, &src_conj, exec)
1183}
1184
1185/// Bytes of one operand below which [`copy_conj_into`] leaves a non-contiguous
1186/// complex copy on the permutation engine ([#467]).
1187///
1188/// Measured crossing, `Complex<f64>`, `Exec::serial()`, ns per element, quoted as
1189/// element map over engine (`> 1` means the engine wins), across four passes:
1190/// 112^3 = 21.4 MiB is 1.51-1.67, 116^3 = 23.8 MiB is 0.93-0.96, 120^3 = 26.4
1191/// MiB is 0.90-0.93, 124^3 = 29.1 MiB is 0.86-0.89 and 128^3 = 32.0 MiB is
1192/// 0.84-0.97. The engine's tiles stop beating the map's blocked traversal once
1193/// one operand no longer fits in a 32 MiB last-level-cache domain, so the bound
1194/// sits inside that bracket at 24 MiB — above the last outright engine win
1195/// (112^3) and below every row the map wins or ties (120^3 and up), with 116^3
1196/// the one tie it leaves on the engine side. `Complex<f32>` is bounded the same
1197/// way in bytes but was not measured.
1198///
1199/// [#467]: https://github.com/tensor4all/strided-rs/issues/467
1200const CONJ_PERM_ENGINE_MAX_BYTES: usize = 24 * 1024 * 1024;
1201
1202/// Whether [`copy_conj_into`] hands this copy to the permutation engine: a
1203/// complex element type, a non-contiguous layout, and a packed footprint below
1204/// [`CONJ_PERM_ENGINE_MAX_BYTES`]. The footprint bound is measured and the other
1205/// two clauses are what the engine's conjugating store cannot do better than the
1206/// map; the table behind all three is `benches/conj_route_467.rs`, recorded in
1207/// `docs/worklogs/2026-11-15-issue-467-conj-route.md`.
1208fn conj_perm_engine_route<T: 'static>(
1209    dims: &[usize],
1210    dst_strides: &[isize],
1211    src_strides: &[isize],
1212) -> bool {
1213    if !element_conj_is_applied::<T>() {
1214        return false;
1215    }
1216    // A contiguous copy takes the map's SIMD store loop (or its splitting
1217    // contiguous path); the engine's conjugating store has no memcpy.
1218    if same_contiguous_layout(dims, &[dst_strides, src_strides]).is_some() {
1219        return false;
1220    }
1221    // An overflowing element count is the map path's `OffsetOverflow` to report.
1222    total_len(dims).is_ok_and(|len| {
1223        len.checked_mul(std::mem::size_of::<T>())
1224            .is_some_and(|bytes| bytes < CONJ_PERM_ENGINE_MAX_BYTES)
1225    })
1226}
1227
1228/// Whether `T`'s `conj` is not the identity, i.e. whether the engine's
1229/// conjugating store does anything.
1230#[inline]
1231fn element_conj_is_applied<T: 'static>() -> bool {
1232    use std::any::TypeId;
1233
1234    let id = TypeId::of::<T>();
1235    id == TypeId::of::<num_complex::Complex32>() || id == TypeId::of::<num_complex::Complex64>()
1236}
1237
1238#[inline]
1239fn element_transpose_is_identity<T: 'static>() -> bool {
1240    use std::any::TypeId;
1241
1242    macro_rules! matches_type {
1243        ($($ty:ty),* $(,)?) => {{
1244            let id = TypeId::of::<T>();
1245            false $(|| id == TypeId::of::<$ty>())*
1246        }};
1247    }
1248
1249    matches_type!(
1250        f32,
1251        f64,
1252        i8,
1253        i16,
1254        i32,
1255        i64,
1256        i128,
1257        isize,
1258        u8,
1259        u16,
1260        u32,
1261        u64,
1262        u128,
1263        usize,
1264        num_complex::Complex32,
1265        num_complex::Complex64,
1266    )
1267}
1268
1269#[inline]
1270fn element_zero_is_all_bits_zero<T: 'static>() -> bool {
1271    use std::any::TypeId;
1272
1273    macro_rules! matches_type {
1274        ($($ty:ty),* $(,)?) => {{
1275            let id = TypeId::of::<T>();
1276            false $(|| id == TypeId::of::<$ty>())*
1277        }};
1278    }
1279
1280    matches_type!(
1281        f32,
1282        f64,
1283        i8,
1284        i16,
1285        i32,
1286        i64,
1287        i128,
1288        isize,
1289        u8,
1290        u16,
1291        u32,
1292        u64,
1293        u128,
1294        usize,
1295        num_complex::Complex32,
1296        num_complex::Complex64,
1297    )
1298}
1299
1300#[inline]
1301unsafe fn fill_2d<T: Copy + Send + Sync>(
1302    dst: *mut T,
1303    dim0: usize,
1304    dim1: usize,
1305    dst_stride0: isize,
1306    dst_stride1: isize,
1307    value: T,
1308    exec: Exec<'_>,
1309) {
1310    let total = dim0.saturating_mul(dim1);
1311    let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
1312    if !exec.is_serial() {
1313        let dst_send = SendPtr(dst);
1314        if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1315            crate::threading::parallel_for_each(0..dim1, exec, &|columns| {
1316                for j in columns {
1317                    let dst = dst_send.as_ptr();
1318                    unsafe {
1319                        let base = j as isize * dst_stride1;
1320                        for i in 0..dim0 {
1321                            *dst.offset(base + i as isize * dst_stride0) = value;
1322                        }
1323                    }
1324                }
1325            });
1326        } else {
1327            crate::threading::parallel_for_each(0..dim0, exec, &|rows| {
1328                for i in rows {
1329                    let dst = dst_send.as_ptr();
1330                    unsafe {
1331                        let base = i as isize * dst_stride0;
1332                        for j in 0..dim1 {
1333                            *dst.offset(base + j as isize * dst_stride1) = value;
1334                        }
1335                    }
1336                }
1337            });
1338        }
1339        return;
1340    }
1341
1342    if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1343        for j in 0..dim1 {
1344            let base = j as isize * dst_stride1;
1345            for i in 0..dim0 {
1346                *dst.offset(base + i as isize * dst_stride0) = value;
1347            }
1348        }
1349    } else {
1350        for i in 0..dim0 {
1351            let base = i as isize * dst_stride0;
1352            for j in 0..dim1 {
1353                *dst.offset(base + j as isize * dst_stride1) = value;
1354            }
1355        }
1356    }
1357}
1358
1359#[inline]
1360unsafe fn fill_contiguous<T>(dst: *mut T, len: usize, value: T)
1361where
1362    T: Copy + Zero + PartialEq + Send + Sync + 'static,
1363{
1364    if element_zero_is_all_bits_zero::<T>() && value == T::zero() {
1365        std::ptr::write_bytes(dst, 0, len);
1366        return;
1367    }
1368
1369    let dst = std::slice::from_raw_parts_mut(dst, len);
1370    dst.fill(value);
1371}
1372
1373// 4x4 transpose block: both base pointers with both strides, the block index and
1374// the scale.
1375#[allow(clippy::too_many_arguments)]
1376#[inline(always)]
1377unsafe fn transpose_scale_4x4_f64(
1378    dst: *mut f64,
1379    dst_stride0: isize,
1380    dst_stride1: isize,
1381    src: *const f64,
1382    src_stride0: isize,
1383    src_stride1: isize,
1384    i: usize,
1385    j: usize,
1386    scale: f64,
1387) {
1388    let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1389
1390    let s00 = *src_base;
1391    let s10 = *src_base.offset(src_stride0);
1392    let s20 = *src_base.offset(2 * src_stride0);
1393    let s30 = *src_base.offset(3 * src_stride0);
1394
1395    let src_col1 = src_base.offset(src_stride1);
1396    let s01 = *src_col1;
1397    let s11 = *src_col1.offset(src_stride0);
1398    let s21 = *src_col1.offset(2 * src_stride0);
1399    let s31 = *src_col1.offset(3 * src_stride0);
1400
1401    let src_col2 = src_base.offset(2 * src_stride1);
1402    let s02 = *src_col2;
1403    let s12 = *src_col2.offset(src_stride0);
1404    let s22 = *src_col2.offset(2 * src_stride0);
1405    let s32 = *src_col2.offset(3 * src_stride0);
1406
1407    let src_col3 = src_base.offset(3 * src_stride1);
1408    let s03 = *src_col3;
1409    let s13 = *src_col3.offset(src_stride0);
1410    let s23 = *src_col3.offset(2 * src_stride0);
1411    let s33 = *src_col3.offset(3 * src_stride0);
1412
1413    let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1414    *dst_row0 = scale * s00;
1415    *dst_row0.offset(dst_stride0) = scale * s01;
1416    *dst_row0.offset(2 * dst_stride0) = scale * s02;
1417    *dst_row0.offset(3 * dst_stride0) = scale * s03;
1418
1419    let dst_row1 = dst_row0.offset(dst_stride1);
1420    *dst_row1 = scale * s10;
1421    *dst_row1.offset(dst_stride0) = scale * s11;
1422    *dst_row1.offset(2 * dst_stride0) = scale * s12;
1423    *dst_row1.offset(3 * dst_stride0) = scale * s13;
1424
1425    let dst_row2 = dst_row0.offset(2 * dst_stride1);
1426    *dst_row2 = scale * s20;
1427    *dst_row2.offset(dst_stride0) = scale * s21;
1428    *dst_row2.offset(2 * dst_stride0) = scale * s22;
1429    *dst_row2.offset(3 * dst_stride0) = scale * s23;
1430
1431    let dst_row3 = dst_row0.offset(3 * dst_stride1);
1432    *dst_row3 = scale * s30;
1433    *dst_row3.offset(dst_stride0) = scale * s31;
1434    *dst_row3.offset(2 * dst_stride0) = scale * s32;
1435    *dst_row3.offset(3 * dst_stride0) = scale * s33;
1436}
1437
1438#[inline]
1439// Tile kernel for f64: both bases, both stride pairs, the tile bounds, the scale and
1440// the token.
1441#[allow(clippy::too_many_arguments)]
1442unsafe fn copy_transpose_scale_2d_f64_tiled_raw(
1443    dst: *mut f64,
1444    dst_stride0: isize,
1445    dst_stride1: isize,
1446    src: *const f64,
1447    src_stride0: isize,
1448    src_stride1: isize,
1449    src_rows: usize,
1450    src_cols: usize,
1451    scale: f64,
1452    exec: Exec<'_>,
1453) {
1454    const TILE: usize = 4;
1455    let row_full = src_rows / TILE * TILE;
1456    let col_full = src_cols / TILE * TILE;
1457
1458    let total = src_rows.saturating_mul(src_cols);
1459    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1460    if !exec.is_serial() {
1461        let dst_send = SendPtr(dst);
1462        let src_send = SendPtr(src as *mut f64);
1463        let row_tiles = row_full / TILE;
1464        crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1465            for tile_i in tiles {
1466                let i = tile_i * TILE;
1467                let dst = dst_send.as_ptr();
1468                let src = src_send.as_const();
1469                unsafe {
1470                    let mut j = 0;
1471                    while j < col_full {
1472                        transpose_scale_4x4_f64(
1473                            dst,
1474                            dst_stride0,
1475                            dst_stride1,
1476                            src,
1477                            src_stride0,
1478                            src_stride1,
1479                            i,
1480                            j,
1481                            scale,
1482                        );
1483                        j += TILE;
1484                    }
1485                    for j in col_full..src_cols {
1486                        for ii in i..i + TILE {
1487                            *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1488                                scale
1489                                    * *src.offset(
1490                                        ii as isize * src_stride0 + j as isize * src_stride1,
1491                                    );
1492                        }
1493                    }
1494                }
1495            }
1496        });
1497        for i in row_full..src_rows {
1498            for j in 0..src_cols {
1499                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1500                    scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1501            }
1502        }
1503        return;
1504    }
1505
1506    let mut i = 0;
1507    while i < row_full {
1508        let mut j = 0;
1509        while j < col_full {
1510            transpose_scale_4x4_f64(
1511                dst,
1512                dst_stride0,
1513                dst_stride1,
1514                src,
1515                src_stride0,
1516                src_stride1,
1517                i,
1518                j,
1519                scale,
1520            );
1521            j += TILE;
1522        }
1523        for j in col_full..src_cols {
1524            for ii in i..i + TILE {
1525                *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1526                    scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1527            }
1528        }
1529        i += TILE;
1530    }
1531    for i in row_full..src_rows {
1532        for j in 0..src_cols {
1533            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1534                scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1535        }
1536    }
1537}
1538
1539#[inline]
1540#[cfg(test)]
1541unsafe fn try_copy_transpose_scale_2d_f64_tiled(
1542    dest: &mut StridedViewMut<f64>,
1543    src: &StridedView<f64>,
1544    scale: f64,
1545    exec: Exec<'_>,
1546) -> bool {
1547    if src.ndim() != 2 || dest.ndim() != 2 {
1548        return false;
1549    }
1550    let src_dims = src.dims();
1551    if dest.dims() != [src_dims[1], src_dims[0]] {
1552        return false;
1553    }
1554    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1555        return false;
1556    }
1557
1558    copy_transpose_scale_2d_f64_tiled_raw(
1559        dest.as_mut_ptr(),
1560        dest.strides()[0],
1561        dest.strides()[1],
1562        src.ptr(),
1563        src.strides()[0],
1564        src.strides()[1],
1565        src_dims[0],
1566        src_dims[1],
1567        scale,
1568        exec,
1569    );
1570    true
1571}
1572
1573#[inline]
1574unsafe fn try_copy_transpose_scale_2d_f64_tiled_typed<T>(
1575    dest: &mut StridedViewMut<T>,
1576    src: &StridedView<T>,
1577    scale: T,
1578    exec: Exec<'_>,
1579) -> bool
1580where
1581    T: Copy + 'static,
1582{
1583    if std::any::TypeId::of::<T>() != std::any::TypeId::of::<f64>() {
1584        return false;
1585    }
1586
1587    let scale = *(&scale as *const T).cast::<f64>();
1588    if src.ndim() != 2 || dest.ndim() != 2 {
1589        return false;
1590    }
1591    let src_dims = src.dims();
1592    if dest.dims() != [src_dims[1], src_dims[0]] {
1593        return false;
1594    }
1595    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1596        return false;
1597    }
1598
1599    copy_transpose_scale_2d_f64_tiled_raw(
1600        dest.as_mut_ptr().cast::<f64>(),
1601        dest.strides()[0],
1602        dest.strides()[1],
1603        src.ptr().cast::<f64>(),
1604        src.strides()[0],
1605        src.strides()[1],
1606        src_dims[0],
1607        src_dims[1],
1608        scale,
1609        exec,
1610    );
1611    true
1612}
1613
1614// Same 4x4 transpose block for the identity element op, which skips the op.
1615#[allow(clippy::too_many_arguments)]
1616#[inline(always)]
1617unsafe fn transpose_scale_4x4_identity<T>(
1618    dst: *mut T,
1619    dst_stride0: isize,
1620    dst_stride1: isize,
1621    src: *const T,
1622    src_stride0: isize,
1623    src_stride1: isize,
1624    i: usize,
1625    j: usize,
1626    scale: T,
1627) where
1628    T: Copy + Mul<Output = T>,
1629{
1630    let src_base = src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1631
1632    let s00 = *src_base;
1633    let s10 = *src_base.offset(src_stride0);
1634    let s20 = *src_base.offset(2 * src_stride0);
1635    let s30 = *src_base.offset(3 * src_stride0);
1636
1637    let src_col1 = src_base.offset(src_stride1);
1638    let s01 = *src_col1;
1639    let s11 = *src_col1.offset(src_stride0);
1640    let s21 = *src_col1.offset(2 * src_stride0);
1641    let s31 = *src_col1.offset(3 * src_stride0);
1642
1643    let src_col2 = src_base.offset(2 * src_stride1);
1644    let s02 = *src_col2;
1645    let s12 = *src_col2.offset(src_stride0);
1646    let s22 = *src_col2.offset(2 * src_stride0);
1647    let s32 = *src_col2.offset(3 * src_stride0);
1648
1649    let src_col3 = src_base.offset(3 * src_stride1);
1650    let s03 = *src_col3;
1651    let s13 = *src_col3.offset(src_stride0);
1652    let s23 = *src_col3.offset(2 * src_stride0);
1653    let s33 = *src_col3.offset(3 * src_stride0);
1654
1655    let dst_row0 = dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1);
1656    *dst_row0 = scale * s00;
1657    *dst_row0.offset(dst_stride0) = scale * s01;
1658    *dst_row0.offset(2 * dst_stride0) = scale * s02;
1659    *dst_row0.offset(3 * dst_stride0) = scale * s03;
1660
1661    let dst_row1 = dst_row0.offset(dst_stride1);
1662    *dst_row1 = scale * s10;
1663    *dst_row1.offset(dst_stride0) = scale * s11;
1664    *dst_row1.offset(2 * dst_stride0) = scale * s12;
1665    *dst_row1.offset(3 * dst_stride0) = scale * s13;
1666
1667    let dst_row2 = dst_row0.offset(2 * dst_stride1);
1668    *dst_row2 = scale * s20;
1669    *dst_row2.offset(dst_stride0) = scale * s21;
1670    *dst_row2.offset(2 * dst_stride0) = scale * s22;
1671    *dst_row2.offset(3 * dst_stride0) = scale * s23;
1672
1673    let dst_row3 = dst_row0.offset(3 * dst_stride1);
1674    *dst_row3 = scale * s30;
1675    *dst_row3.offset(dst_stride0) = scale * s31;
1676    *dst_row3.offset(2 * dst_stride0) = scale * s32;
1677    *dst_row3.offset(3 * dst_stride0) = scale * s33;
1678}
1679
1680#[inline]
1681// Tile kernel: both bases, both stride pairs, the tile bounds, the scale and the token.
1682#[allow(clippy::too_many_arguments)]
1683unsafe fn copy_transpose_scale_2d_identity_tiled_raw<T>(
1684    dst: *mut T,
1685    dst_stride0: isize,
1686    dst_stride1: isize,
1687    src: *const T,
1688    src_stride0: isize,
1689    src_stride1: isize,
1690    src_rows: usize,
1691    src_cols: usize,
1692    scale: T,
1693    exec: Exec<'_>,
1694) where
1695    T: Copy + Mul<Output = T> + Send + Sync,
1696{
1697    const TILE: usize = 4;
1698    let row_full = src_rows / TILE * TILE;
1699    let col_full = src_cols / TILE * TILE;
1700
1701    let total = src_rows.saturating_mul(src_cols);
1702    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1703    if !exec.is_serial() {
1704        let dst_send = SendPtr(dst);
1705        let src_send = SendPtr(src as *mut T);
1706        let row_tiles = row_full / TILE;
1707        crate::threading::parallel_for_each(0..row_tiles, exec, &|tiles| {
1708            for tile_i in tiles {
1709                let i = tile_i * TILE;
1710                let dst = dst_send.as_ptr();
1711                let src = src_send.as_const();
1712                unsafe {
1713                    let mut j = 0;
1714                    while j < col_full {
1715                        transpose_scale_4x4_identity(
1716                            dst,
1717                            dst_stride0,
1718                            dst_stride1,
1719                            src,
1720                            src_stride0,
1721                            src_stride1,
1722                            i,
1723                            j,
1724                            scale,
1725                        );
1726                        j += TILE;
1727                    }
1728                    for j in col_full..src_cols {
1729                        for ii in i..i + TILE {
1730                            *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1731                                scale
1732                                    * *src.offset(
1733                                        ii as isize * src_stride0 + j as isize * src_stride1,
1734                                    );
1735                        }
1736                    }
1737                }
1738            }
1739        });
1740        for i in row_full..src_rows {
1741            for j in 0..src_cols {
1742                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1743                    scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1744            }
1745        }
1746        return;
1747    }
1748
1749    let mut i = 0;
1750    while i < row_full {
1751        let mut j = 0;
1752        while j < col_full {
1753            transpose_scale_4x4_identity(
1754                dst,
1755                dst_stride0,
1756                dst_stride1,
1757                src,
1758                src_stride0,
1759                src_stride1,
1760                i,
1761                j,
1762                scale,
1763            );
1764            j += TILE;
1765        }
1766        for j in col_full..src_cols {
1767            for ii in i..i + TILE {
1768                *dst.offset(j as isize * dst_stride0 + ii as isize * dst_stride1) =
1769                    scale * *src.offset(ii as isize * src_stride0 + j as isize * src_stride1);
1770            }
1771        }
1772        i += TILE;
1773    }
1774    for i in row_full..src_rows {
1775        for j in 0..src_cols {
1776            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1777                scale * *src.offset(i as isize * src_stride0 + j as isize * src_stride1);
1778        }
1779    }
1780}
1781
1782#[inline]
1783unsafe fn try_copy_transpose_scale_2d_identity_tiled<T>(
1784    dest: &mut StridedViewMut<T>,
1785    src: &StridedView<T>,
1786    scale: T,
1787    exec: Exec<'_>,
1788) -> bool
1789where
1790    T: Copy + Mul<Output = T> + Send + Sync,
1791{
1792    if src.ndim() != 2 || dest.ndim() != 2 {
1793        return false;
1794    }
1795    let src_dims = src.dims();
1796    if dest.dims() != [src_dims[1], src_dims[0]] {
1797        return false;
1798    }
1799    if src.strides()[0] != 1 || dest.strides()[0] != 1 {
1800        return false;
1801    }
1802
1803    copy_transpose_scale_2d_identity_tiled_raw(
1804        dest.as_mut_ptr(),
1805        dest.strides()[0],
1806        dest.strides()[1],
1807        src.ptr(),
1808        src.strides()[0],
1809        src.strides()[1],
1810        src_dims[0],
1811        src_dims[1],
1812        scale,
1813        exec,
1814    );
1815    true
1816}
1817
1818#[inline]
1819// Same signature as the tiled kernels, so the loop and tiled paths stay comparable.
1820#[allow(clippy::too_many_arguments)]
1821unsafe fn copy_transpose_scale_2d_loop<T>(
1822    dst: *mut T,
1823    dst_stride0: isize,
1824    dst_stride1: isize,
1825    src: *const T,
1826    src_stride0: isize,
1827    src_stride1: isize,
1828    src_rows: usize,
1829    src_cols: usize,
1830    scale: T,
1831    exec: Exec<'_>,
1832) where
1833    T: Copy + ElementOpApply + Mul<Output = T> + Send + Sync,
1834{
1835    let total = src_rows.saturating_mul(src_cols);
1836    let exec = crate::cost::for_len(exec, KernelClass::Map, total);
1837    if !exec.is_serial() {
1838        let dst_send = SendPtr(dst);
1839        let src_send = SendPtr(src as *mut T);
1840        if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1841            crate::threading::parallel_for_each(0..src_rows, exec, &|rows| {
1842                for i in rows {
1843                    let dst = dst_send.as_ptr();
1844                    let src = src_send.as_const();
1845                    unsafe {
1846                        for j in 0..src_cols {
1847                            let value = (*src
1848                                .offset(i as isize * src_stride0 + j as isize * src_stride1))
1849                            .transpose();
1850                            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1851                                scale * value;
1852                        }
1853                    }
1854                }
1855            });
1856        } else {
1857            crate::threading::parallel_for_each(0..src_cols, exec, &|columns| {
1858                for j in columns {
1859                    let dst = dst_send.as_ptr();
1860                    let src = src_send.as_const();
1861                    unsafe {
1862                        for i in 0..src_rows {
1863                            let value = (*src
1864                                .offset(i as isize * src_stride0 + j as isize * src_stride1))
1865                            .transpose();
1866                            *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) =
1867                                scale * value;
1868                        }
1869                    }
1870                }
1871            });
1872        }
1873        return;
1874    }
1875
1876    if dst_stride0.unsigned_abs() <= dst_stride1.unsigned_abs() {
1877        for i in 0..src_rows {
1878            for j in 0..src_cols {
1879                let value =
1880                    (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1881                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1882            }
1883        }
1884    } else {
1885        for j in 0..src_cols {
1886            for i in 0..src_rows {
1887                let value =
1888                    (*src.offset(i as isize * src_stride0 + j as isize * src_stride1)).transpose();
1889                *dst.offset(j as isize * dst_stride0 + i as isize * dst_stride1) = scale * value;
1890            }
1891        }
1892    }
1893}
1894
1895/// Copy with transpose and scaling: `dest[j,i] = scale * src[i,j]`.
1896///
1897/// # Errors
1898///
1899/// Returns [`StridedError::RankMismatch`] when `src` or `dest` is not rank
1900/// 2, [`StridedError::ShapeMismatch`] when `dest` does not have the
1901/// transposed extents of `src`, and
1902/// [`StridedError::OffsetOverflow`] when the element count does not fit in
1903/// `usize`. When `scale` is one and `T`'s transpose is the identity, the
1904/// permutation path also returns [`StridedError::OffsetOverflow`] when the
1905/// layout cannot be expressed in `isize` stride arithmetic or a fused extent
1906/// overflows, and [`StridedError::NonInjectiveOutputLayout`] when the
1907/// destination layout cannot be proven representable and injective.
1908pub fn copy_transpose_scale_into<T>(
1909    dest: &mut StridedViewMut<T>,
1910    src: &StridedView<T>,
1911    scale: T,
1912    exec: Exec<'_>,
1913) -> Result<()>
1914where
1915    T: Copy + ElementOpApply + Mul<Output = T> + Zero + One + PartialEq + Send + Sync + 'static,
1916{
1917    let mut exec = exec;
1918    if src.ndim() != 2 || dest.ndim() != 2 {
1919        return Err(StridedError::RankMismatch(src.ndim(), 2));
1920    }
1921
1922    let src_dims = src.dims();
1923    let expected_dims = [src_dims[1], src_dims[0]];
1924    ensure_same_shape(dest.dims(), &expected_dims)?;
1925    // Reject an element count beyond usize (a huge stride-0 broadcast) up
1926    // front instead of looping over it.
1927    total_len(src_dims)?;
1928
1929    if scale == T::zero() {
1930        unsafe {
1931            if same_contiguous_layout(dest.dims(), &[dest.strides()]).is_some() {
1932                fill_contiguous(dest.as_mut_ptr(), total_len(dest.dims())?, T::zero());
1933            } else {
1934                fill_2d(
1935                    dest.as_mut_ptr(),
1936                    dest.dims()[0],
1937                    dest.dims()[1],
1938                    dest.strides()[0],
1939                    dest.strides()[1],
1940                    T::zero(),
1941                    exec,
1942                );
1943            }
1944        }
1945        return Ok(());
1946    }
1947
1948    let transpose_is_identity = element_transpose_is_identity::<T>();
1949
1950    unsafe {
1951        if transpose_is_identity
1952            && try_copy_transpose_scale_2d_f64_tiled_typed(dest, src, scale, exec.reborrow())
1953        {
1954            return Ok(());
1955        }
1956        if transpose_is_identity
1957            && try_copy_transpose_scale_2d_identity_tiled(dest, src, scale, exec.reborrow())
1958        {
1959            return Ok(());
1960        }
1961    }
1962
1963    if scale == T::one() && transpose_is_identity {
1964        let src_t = src.permute(&[1, 0])?;
1965        return crate::threading::copy_permuted(dest, &src_t, exec);
1966    }
1967
1968    unsafe {
1969        copy_transpose_scale_2d_loop(
1970            dest.as_mut_ptr(),
1971            dest.strides()[0],
1972            dest.strides()[1],
1973            src.ptr(),
1974            src.strides()[0],
1975            src.strides()[1],
1976            src_dims[0],
1977            src_dims[1],
1978            scale,
1979            exec,
1980        );
1981    }
1982    Ok(())
1983}
1984
1985#[cfg(test)]
1986#[path = "ops_view/tests/tiled_tests.rs"]
1987mod tiled_tests;
1988
1989#[cfg(test)]
1990#[path = "ops_view/tests/conj_route_tests.rs"]
1991mod conj_route_tests;