Skip to main content

strided_basic/
copy_plan.rs

1//! Prepared (compile-once, execute-many) copy plans over raw strided layouts.
2//!
3//! [`copy_scale_raw`](crate::copy_scale_raw) and friends rebuild the fused
4//! loop nest on every call; for prepared-replay consumers that issue many
5//! small copies with a fixed layout, that per-call planning dominates
6//! (see issue #139). [`CopyPlan`] splits the work: [`CopyPlan::compile`]
7//! validates the layout pair and builds the fused traversal once,
8//! [`CopyPlan::execute`]/[`CopyPlan::execute_scale`]/[`CopyPlan::execute_conj`]
9//! replay it with no planning and no heap allocation for fused ranks at most
10//! [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
11
12use core::{
13    marker::PhantomData,
14    mem::MaybeUninit,
15    ops::{Add, Mul},
16};
17
18use num_traits::{One, Zero};
19
20use crate::cost::KernelClass;
21use crate::raw_ops::{
22    apply_fused_range, apply_spilled_range, fuse_pair_traversal, fused_total, FusedAxis,
23    FusedPairLayout, PairTraversal,
24};
25use crate::{ElementOpApply, Exec, RawStridedMut, RawStridedRef, Result, StridedError};
26
27// Same pattern as map_view.rs / outer_product.rs: inline storage in both
28// feature configurations. Only `compile` touches these; `execute*` never
29// allocates for them. (A spilled fused layout needs one cursor allocation per
30// replayed range, so an `execute*` can allocate once per worker range above the
31// inline rank bound.)
32type AxisVec<T> = crate::inline::InlineVec<T>;
33
34pub(crate) trait OverwriteWriter<T> {
35    fn dims(&self) -> &[usize];
36    fn strides(&self) -> &[isize];
37    fn offset(&self) -> isize;
38    /// # Safety
39    /// The caller must use the pointer only for the validated allocation and
40    /// logical layout represented by this writer.
41    unsafe fn data_ptr(&mut self) -> *mut T;
42    /// # Safety
43    /// The offset must be an in-bounds logical destination proven by layout.
44    unsafe fn write_at(&mut self, offset: isize, value: T);
45}
46
47pub(crate) trait ReadModifyWrite<T>: OverwriteWriter<T> {
48    /// # Safety
49    /// The offset must be an in-bounds initialized slot covered by the
50    /// traversal's copy and disjointness proof.
51    unsafe fn add_at<F>(&mut self, offset: isize, value: T, combine: F)
52    where
53        F: FnOnce(T, T) -> T;
54}
55
56impl<'a, T> OverwriteWriter<T> for RawStridedMut<'a, T> {
57    fn dims(&self) -> &[usize] {
58        self.dims()
59    }
60    fn strides(&self) -> &[isize] {
61        self.strides()
62    }
63    fn offset(&self) -> isize {
64        self.offset()
65    }
66    unsafe fn data_ptr(&mut self) -> *mut T {
67        self.data_mut().as_mut_ptr()
68    }
69    unsafe fn write_at(&mut self, offset: isize, value: T) {
70        // SAFETY: the prepared layout validates every logical destination.
71        unsafe { self.data_mut().as_mut_ptr().offset(offset).write(value) }
72    }
73}
74
75impl<'a, T> ReadModifyWrite<T> for RawStridedMut<'a, T>
76where
77    T: Add<Output = T>,
78{
79    #[inline(always)]
80    unsafe fn add_at<F>(&mut self, offset: isize, value: T, combine: F)
81    where
82        F: FnOnce(T, T) -> T,
83    {
84        // SAFETY: the copy or initialized caller proves this logical slot.
85        unsafe {
86            let ptr = self.data_mut().as_mut_ptr().offset(offset);
87            ptr.write(combine(ptr.read(), value));
88        }
89    }
90}
91
92impl<'a, T> OverwriteWriter<T> for RawStridedMut<'a, MaybeUninit<T>> {
93    fn dims(&self) -> &[usize] {
94        self.dims()
95    }
96    fn strides(&self) -> &[isize] {
97        self.strides()
98    }
99    fn offset(&self) -> isize {
100        self.offset()
101    }
102    unsafe fn data_ptr(&mut self) -> *mut T {
103        self.data_mut().as_mut_ptr().cast()
104    }
105    unsafe fn write_at(&mut self, offset: isize, value: T) {
106        // SAFETY: the prepared layout validates every logical destination.
107        unsafe {
108            self.data_mut()
109                .as_mut_ptr()
110                .offset(offset)
111                .write(MaybeUninit::new(value))
112        }
113    }
114}
115
116pub(crate) struct InitializedRawDest<'a, T> {
117    ptr: *mut T,
118    extent: usize,
119    dims: &'a [usize],
120    strides: &'a [isize],
121    offset: isize,
122    _marker: PhantomData<&'a mut [MaybeUninit<T>]>,
123}
124
125impl<'a, T> OverwriteWriter<T> for InitializedRawDest<'a, T> {
126    fn dims(&self) -> &[usize] {
127        self.dims
128    }
129    fn strides(&self) -> &[isize] {
130        self.strides
131    }
132    fn offset(&self) -> isize {
133        self.offset
134    }
135    unsafe fn data_ptr(&mut self) -> *mut T {
136        self.ptr
137    }
138    unsafe fn write_at(&mut self, offset: isize, value: T) {
139        debug_assert!(offset >= 0 && (offset as usize) < self.extent);
140        // SAFETY: the copy proof and extent check cover this logical slot.
141        unsafe { self.ptr.offset(offset).write(value) }
142    }
143}
144
145impl<'a, T> ReadModifyWrite<T> for InitializedRawDest<'a, T>
146where
147    T: Add<Output = T>,
148{
149    #[inline(always)]
150    unsafe fn add_at<F>(&mut self, offset: isize, value: T, combine: F)
151    where
152        F: FnOnce(T, T) -> T,
153    {
154        debug_assert!(offset >= 0 && (offset as usize) < self.extent);
155        // SAFETY: the copy proof and extent check cover this logical slot.
156        unsafe {
157            let ptr = self.ptr.offset(offset);
158            ptr.write(combine(ptr.read(), value));
159        }
160    }
161}
162
163/// A compiled copy traversal for one `(dims, dst_strides, src_strides)`
164/// layout pair.
165///
166/// `compile` proves the layout facts once (rank agreement, extent overflow,
167/// destination injectivity) and fuses/orders the loop nest; each `execute*`
168/// call then only re-checks the per-call facts (that the supplied views carry
169/// exactly the compiled layout) before replaying the prepared loops.
170///
171/// Overlapping `src`/`dest` memory is not supported, matching the rest of the
172/// crate.
173///
174/// Ranks above [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT) are supported by the same fused
175/// traversal, which then stores its axes in one rank-bounded allocation.
176///
177/// # Example
178///
179/// ```rust
180/// use strided_basic::{CopyPlan, Exec, RawStridedMut, RawStridedRef};
181///
182/// let dims = [2usize, 3];
183/// let src_strides = [3isize, 1];
184/// let dst_strides = [1isize, 2]; // transposed destination
185/// let plan = CopyPlan::compile(&dims, &dst_strides, &src_strides).unwrap();
186///
187/// let src = [0.0f64, 1.0, 2.0, 10.0, 11.0, 12.0];
188/// let mut dst = [0.0f64; 6];
189/// let src_ref = RawStridedRef::new(&src, &dims, &src_strides, 0).unwrap();
190/// let mut dst_mut = RawStridedMut::new(&mut dst, &dims, &dst_strides, 0).unwrap();
191/// plan.execute(&mut dst_mut, &src_ref, Exec::serial()).unwrap();
192/// assert_eq!(dst, [0.0, 10.0, 1.0, 11.0, 2.0, 12.0]);
193/// ```
194#[derive(Clone, Debug, Default)]
195pub struct CopyPlan {
196    dims: AxisVec<usize>,
197    dst_strides: AxisVec<isize>,
198    src_strides: AxisVec<isize>,
199    fused: PairTraversal,
200}
201
202impl CopyPlan {
203    /// `None` when the fused traversal is stored in the rank-bounded spill.
204    #[inline]
205    pub(crate) fn fused_layout(&self) -> Option<&FusedPairLayout> {
206        self.fused.inline()
207    }
208
209    /// Copy into uninitialized storage, then lend the initialized destination to `f`.
210    ///
211    /// [`Self::execute_uninit`] initializes every reachable logical destination
212    /// element before `f` runs, so the [`InitializedRawDest`] receipt handed to `f`
213    /// may read and overwrite exactly those offsets. Slots outside the plan's
214    /// reachable set are not written by that copy and it makes no claim about
215    /// them: a caller that handed in a `MaybeUninit` array still owns those slots,
216    /// so they keep whatever initialization the caller left. With
217    /// `CopyPlan::compile(&[2], &[2], &[1])` the offset-1 hole keeps the caller's
218    /// value (measured `[7.0, 99.0, 8.0]` for `src = [7, 8]`) rather than
219    /// becoming uninitialized. That is why the receipt exposes
220    /// `write_at`/`add_at` (copy_plan.rs:138, :150) instead of a typed slice: their
221    /// contracts require an offset proven by this plan's layout. The receipt borrows
222    /// the destination for `f`'s lifetime, so a read-modify-write consumer (the
223    /// gather and scatter update replay in gather_plan.rs) sees the copied operand
224    /// once and cannot alias the source.
225    ///
226    /// # Errors
227    ///
228    /// Returns the [`Self::execute_uninit`] errors (`PlanLayoutMismatch`, before any
229    /// write); `f` runs only on success.
230    pub(crate) fn execute_uninit_then<'a, T, R>(
231        &self,
232        dest: &'a mut RawStridedMut<'a, MaybeUninit<T>>,
233        src: &RawStridedRef<'_, T>,
234        mut exec: Exec<'_>,
235        f: impl for<'b, 'e> FnOnce(InitializedRawDest<'b, T>, Exec<'e>) -> R,
236    ) -> Result<R>
237    where
238        T: Copy + Send + Sync,
239    {
240        self.execute_uninit(dest, src, exec.reborrow())?;
241        let data = dest.data_mut();
242        let receipt = InitializedRawDest {
243            ptr: data.as_mut_ptr().cast(),
244            extent: data.len(),
245            dims: dest.dims(),
246            strides: dest.strides(),
247            offset: dest.offset(),
248            _marker: PhantomData,
249        };
250        Ok(f(receipt, exec))
251    }
252
253    /// Compile a copy plan for the given layout pair.
254    ///
255    /// Performs the layout validation and traversal construction
256    /// (fuse + order) once.
257    ///
258    /// # Errors
259    ///
260    /// Returns [`StridedError::StrideLengthMismatch`] when `dims`,
261    /// `dst_strides`, and `src_strides` do not all have the same length,
262    /// [`StridedError::OffsetOverflow`] when the total element count does not
263    /// fit `usize`, and [`StridedError::NonInjectiveOutputLayout`] when the
264    /// destination layout cannot be proven injective (aliasing elements, or a
265    /// span the check cannot represent).
266    pub fn compile(dims: &[usize], dst_strides: &[isize], src_strides: &[isize]) -> Result<Self> {
267        if dims.len() != dst_strides.len() || dims.len() != src_strides.len() {
268            return Err(StridedError::StrideLengthMismatch);
269        }
270        // A zero-sized layout has zero elements even when its other extents
271        // overflow; only a nonempty overflowing count is rejected.
272        crate::kernel::total_len(dims)?;
273        if !crate::layout_check::is_injective_layout(dims, dst_strides) {
274            return Err(StridedError::NonInjectiveOutputLayout);
275        }
276        Ok(Self {
277            dims: dims.into(),
278            dst_strides: dst_strides.into(),
279            src_strides: src_strides.into(),
280            fused: fuse_pair_traversal(dims, dst_strides, src_strides),
281        })
282    }
283
284    /// Check the per-call facts: the supplied views must carry exactly the
285    /// compiled layout. Buffer bounds against that layout were already proven
286    /// by [`RawStridedRef::new`]/[`RawStridedMut::new`] (or asserted by the
287    /// caller of the `new_unchecked` constructors), so layout equality is the
288    /// complete precondition for the pointer-based fused replay below.
289    fn check_call<D, S>(
290        &self,
291        dest: &RawStridedMut<'_, D>,
292        src: &RawStridedRef<'_, S>,
293    ) -> Result<()> {
294        check_layout_pair(&self.dims, &self.dst_strides, &self.src_strides, dest, src)
295    }
296
297    /// Check the per-call facts, then replay the compiled traversal.
298    fn replay<D, S, Apply>(
299        &self,
300        dest: &mut RawStridedMut<'_, D>,
301        src: &RawStridedRef<'_, S>,
302        apply: Apply,
303        exec: Exec<'_>,
304    ) -> Result<()>
305    where
306        D: Copy + Send + Sync,
307        S: Copy + Send + Sync,
308        Apply: Fn(&mut D, S) + Sync,
309    {
310        self.check_call(dest, src)?;
311        // SAFETY: `check_call` proved both views carry exactly the compiled
312        // layout, and `fused` is the fusion of that layout (only `compile`
313        // and `Default`, the rank-zero layout, build it). `compile` checked
314        // the element count and proved the destination injective.
315        unsafe { replay_fused(dest, src, &self.fused, apply, exec) };
316        Ok(())
317    }
318
319    /// `dest = src` into a potentially uninitialized destination.
320    ///
321    /// On success every reachable logical destination element is initialized.
322    /// Non-reachable holes in the backing allocation are not written.
323    ///
324    /// # Errors
325    ///
326    /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
327    /// carry the compiled layout, before any write.
328    pub fn execute_uninit<T>(
329        &self,
330        dest: &mut RawStridedMut<'_, MaybeUninit<T>>,
331        src: &RawStridedRef<'_, T>,
332        exec: Exec<'_>,
333    ) -> Result<()>
334    where
335        T: Copy + Send + Sync,
336    {
337        self.replay(
338            dest,
339            src,
340            |dst: &mut MaybeUninit<T>, value: T| {
341                dst.write(value);
342            },
343            exec,
344        )
345    }
346
347    /// `dest = src`. Allocation-free for fused ranks at most
348    /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
349    ///
350    /// # Errors
351    ///
352    /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
353    /// carry the compiled layout, before any write.
354    pub fn execute<T>(
355        &self,
356        dest: &mut RawStridedMut<'_, T>,
357        src: &RawStridedRef<'_, T>,
358        exec: Exec<'_>,
359    ) -> Result<()>
360    where
361        T: Copy + Send + Sync,
362    {
363        self.replay(dest, src, |dst: &mut T, value: T| *dst = value, exec)
364    }
365
366    /// `dest = scale * src`. Allocation-free for fused ranks at most
367    /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
368    ///
369    /// # Errors
370    ///
371    /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
372    /// carry the compiled layout, before any write.
373    pub fn execute_scale<T>(
374        &self,
375        dest: &mut RawStridedMut<'_, T>,
376        src: &RawStridedRef<'_, T>,
377        scale: T,
378        exec: Exec<'_>,
379    ) -> Result<()>
380    where
381        T: Copy + Mul<T, Output = T> + Send + Sync,
382    {
383        self.replay(
384            dest,
385            src,
386            |dst: &mut T, value: T| *dst = scale * value,
387            exec,
388        )
389    }
390
391    /// `dest = conj(src)`. Allocation-free for fused ranks at most
392    /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
393    ///
394    /// # Errors
395    ///
396    /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
397    /// carry the compiled layout, before any write.
398    pub fn execute_conj<T>(
399        &self,
400        dest: &mut RawStridedMut<'_, T>,
401        src: &RawStridedRef<'_, T>,
402        exec: Exec<'_>,
403    ) -> Result<()>
404    where
405        T: Copy + ElementOpApply + Send + Sync,
406    {
407        self.replay(dest, src, |dst: &mut T, value: T| *dst = value.conj(), exec)
408    }
409}
410
411/// Verify that a pair of raw views carries exactly a compiled layout.
412///
413/// Shared by [`CopyPlan`] and [`UpdatePlan`], whose `compile` methods prove the
414/// layout facts once. Buffer bounds against that layout were already proven by
415/// [`RawStridedRef::new`]/[`RawStridedMut::new`] (or asserted by the caller of
416/// the `new_unchecked` constructors), so layout equality is the complete
417/// per-call precondition for the pointer-based fused replay.
418fn check_layout_pair<D, S>(
419    dims: &[usize],
420    dst_strides: &[isize],
421    src_strides: &[isize],
422    dest: &RawStridedMut<'_, D>,
423    src: &RawStridedRef<'_, S>,
424) -> Result<()> {
425    if dest.dims() != dims
426        || src.dims() != dims
427        || dest.strides() != dst_strides
428        || src.strides() != src_strides
429    {
430        return Err(StridedError::PlanLayoutMismatch);
431    }
432    Ok(())
433}
434
435/// A compiled read-modify-write traversal for one `(dims, dst_strides,
436/// src_strides)` layout pair.
437///
438/// [`execute`](Self::execute) computes `dest = alpha * f(src) + beta * dest` by
439/// replaying the same fused pair traversal as [`CopyPlan`]: `compile` proves
440/// the layout facts once (rank agreement, extent overflow, destination
441/// injectivity) and each call only re-checks that the supplied views carry the
442/// compiled layout before running monomorphized loops. Execute is
443/// allocation-free for fused ranks at most
444/// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT), and the injective
445/// destination makes a large replay parallel-safe exactly as in [`CopyPlan`].
446///
447/// Overlapping `src`/`dest` memory is not supported, matching the rest of the
448/// crate.
449///
450/// # Example
451///
452/// ```rust
453/// use strided_basic::{Exec, RawStridedMut, RawStridedRef, UpdatePlan};
454///
455/// let dims = [2usize, 2];
456/// let dst_strides = [1isize, 2];
457/// let src_strides = [1isize, 2];
458/// let plan = UpdatePlan::compile(&dims, &dst_strides, &src_strides).unwrap();
459///
460/// let src = [1.0f64, 2.0, 3.0, 4.0];
461/// let mut dst = [10.0f64, 20.0, 30.0, 40.0];
462/// let src_ref = RawStridedRef::new(&src, &dims, &src_strides, 0).unwrap();
463/// let mut dst_mut = RawStridedMut::new(&mut dst, &dims, &dst_strides, 0).unwrap();
464/// // dst = 2 * src + 3 * dst
465/// plan.execute(&mut dst_mut, &src_ref, 2.0, 3.0, |x| x, Exec::serial()).unwrap();
466/// assert_eq!(dst, [32.0, 64.0, 96.0, 128.0]);
467/// ```
468#[derive(Clone, Debug)]
469pub struct UpdatePlan {
470    dims: AxisVec<usize>,
471    dst_strides: AxisVec<isize>,
472    src_strides: AxisVec<isize>,
473    fused: PairTraversal,
474}
475
476impl UpdatePlan {
477    /// Compile an update plan for the given layout pair.
478    ///
479    /// Performs the same layout validation as [`CopyPlan::compile`].
480    ///
481    /// # Errors
482    ///
483    /// Returns [`StridedError::StrideLengthMismatch`] when `dims`,
484    /// `dst_strides`, and `src_strides` do not all have the same length,
485    /// [`StridedError::OffsetOverflow`] when the total element count does not
486    /// fit `usize`, and [`StridedError::NonInjectiveOutputLayout`] when the
487    /// destination layout cannot be proven injective: two logical indices name
488    /// one slot (so a parallel replay would write that slot once per index), or
489    /// the offset span cannot be represented, leaving injectivity undecided.
490    pub fn compile(dims: &[usize], dst_strides: &[isize], src_strides: &[isize]) -> Result<Self> {
491        if dims.len() != dst_strides.len() || dims.len() != src_strides.len() {
492            return Err(StridedError::StrideLengthMismatch);
493        }
494        crate::kernel::total_len(dims)?;
495        if !crate::layout_check::is_injective_layout(dims, dst_strides) {
496            return Err(StridedError::NonInjectiveOutputLayout);
497        }
498        Ok(Self {
499            dims: dims.into(),
500            dst_strides: dst_strides.into(),
501            src_strides: src_strides.into(),
502            fused: fuse_pair_traversal(dims, dst_strides, src_strides),
503        })
504    }
505
506    /// Check the per-call facts, then replay the compiled traversal.
507    fn replay<D, S, Apply>(
508        &self,
509        dest: &mut RawStridedMut<'_, D>,
510        src: &RawStridedRef<'_, S>,
511        apply: Apply,
512        exec: Exec<'_>,
513    ) -> Result<()>
514    where
515        D: Copy + Send + Sync,
516        S: Copy + Send + Sync,
517        Apply: Fn(&mut D, S) + Sync,
518    {
519        check_layout_pair(&self.dims, &self.dst_strides, &self.src_strides, dest, src)?;
520        // SAFETY: as in `CopyPlan::replay`: the views carry the compiled
521        // layout, `fused` is its fusion (built only by `compile`), and
522        // `compile` checked the count and destination injectivity.
523        unsafe { replay_fused(dest, src, &self.fused, apply, exec) };
524        Ok(())
525    }
526
527    /// `dest = alpha * f(src) + beta * dest`.
528    ///
529    /// `f` is applied to the source element before scaling. When `beta` is
530    /// zero (`-0.0` included; `NaN` is not zero) the replay is write-only: the
531    /// old destination element is never read, so a NaN-filled or uninitialized
532    /// destination cannot contaminate the result, matching BLAS `beta = 0`
533    /// semantics. Otherwise the old destination element is read through the
534    /// destination layout. Allocation-free for fused ranks at most
535    /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
536    ///
537    /// # Errors
538    ///
539    /// [`StridedError::PlanLayoutMismatch`] when the supplied views do not
540    /// carry the compiled layout, before any write.
541    pub fn execute<T, F>(
542        &self,
543        dest: &mut RawStridedMut<'_, T>,
544        src: &RawStridedRef<'_, T>,
545        alpha: T,
546        beta: T,
547        f: F,
548        exec: Exec<'_>,
549    ) -> Result<()>
550    where
551        T: Copy + Send + Sync + Add<Output = T> + Mul<Output = T> + Zero + One + PartialEq,
552        F: Fn(T) -> T + Send + Sync,
553    {
554        if beta.is_zero() {
555            // Write-only: never form a read of the old destination.
556            if alpha.is_one() {
557                self.replay(dest, src, |dst: &mut T, value: T| *dst = f(value), exec)
558            } else {
559                self.replay(
560                    dest,
561                    src,
562                    |dst: &mut T, value: T| *dst = alpha * f(value),
563                    exec,
564                )
565            }
566        } else {
567            self.replay(
568                dest,
569                src,
570                |dst: &mut T, value: T| *dst = alpha * f(value) + beta * *dst,
571                exec,
572            )
573        }
574    }
575
576    /// `dest = alpha * f(src)` into a potentially uninitialized destination.
577    ///
578    /// This is [`execute`](Self::execute) with `beta = 0`: the destination is
579    /// never read. On success every reachable logical destination element is
580    /// initialized; non-reachable holes in the backing allocation are not
581    /// written. Allocation-free for fused ranks at most
582    /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
583    ///
584    /// # Errors
585    ///
586    /// [`StridedError::PlanLayoutMismatch`] when the supplied views do not
587    /// carry the compiled layout, before any write.
588    ///
589    /// # Examples
590    ///
591    /// ```rust
592    /// use core::mem::MaybeUninit;
593    /// use strided_basic::{Exec, RawStridedMut, RawStridedRef, UpdatePlan};
594    ///
595    /// let dims = [2usize, 2];
596    /// let strides = [1isize, 2];
597    /// let plan = UpdatePlan::compile(&dims, &strides, &strides).unwrap();
598    /// let src = [1.0f64, 2.0, 3.0, 4.0];
599    /// let mut storage = [MaybeUninit::<f64>::uninit(); 4];
600    /// let mut dst = RawStridedMut::new(&mut storage, &dims, &strides, 0).unwrap();
601    /// let source = RawStridedRef::new(&src, &dims, &strides, 0).unwrap();
602    /// // dst = 2 * src
603    /// plan.execute_uninit(&mut dst, &source, 2.0, |x| x, Exec::serial()).unwrap();
604    /// drop(dst);
605    /// // SAFETY: every reachable destination slot was initialized above.
606    /// let values: Vec<f64> = storage.iter().map(|v| unsafe { v.assume_init() }).collect();
607    /// assert_eq!(values, [2.0, 4.0, 6.0, 8.0]);
608    /// ```
609    pub fn execute_uninit<T, F>(
610        &self,
611        dest: &mut RawStridedMut<'_, MaybeUninit<T>>,
612        src: &RawStridedRef<'_, T>,
613        alpha: T,
614        f: F,
615        exec: Exec<'_>,
616    ) -> Result<()>
617    where
618        T: Copy + Send + Sync + Mul<Output = T> + One + PartialEq,
619        F: Fn(T) -> T + Send + Sync,
620    {
621        if alpha.is_one() {
622            self.replay(
623                dest,
624                src,
625                |dst: &mut MaybeUninit<T>, value: T| {
626                    dst.write(f(value));
627                },
628                exec,
629            )
630        } else {
631            self.replay(
632                dest,
633                src,
634                |dst: &mut MaybeUninit<T>, value: T| {
635                    dst.write(alpha * f(value));
636                },
637                exec,
638            )
639        }
640    }
641}
642
643/// Replay `traversal` over a pair of raw views, splitting it across the budget
644/// of `exec` when the repository threshold allows it.
645///
646/// The logical index space `0..total` (column-major over the traversal axes)
647/// is split into contiguous worker ranges, which divides the outer axes and,
648/// when there are fewer outer positions than workers, also chunks long inner
649/// runs (for example a rank-1 contiguous copy).
650///
651/// # Safety
652///
653/// - `traversal` must replay exactly the logical index space of the two
654///   views: either their dims and strides fused by `fuse_pair_traversal`, or
655///   built from them by `PairTraversal::given`. The views' own bounds proofs
656///   ([`RawStridedRef::new`], [`RawStridedMut::new`], or the caller of their
657///   `new_unchecked`) then make every replayed offset in-bounds.
658/// - The product of the traversal extents must not overflow `usize` (a
659///   checked count, zero included).
660/// - Unless `exec.is_serial()`, the destination strides must be injective
661///   over the traversal.
662/// - The destination and source must not overlap.
663pub(crate) unsafe fn replay_fused<D, S, Apply>(
664    dest: &mut RawStridedMut<'_, D>,
665    src: &RawStridedRef<'_, S>,
666    traversal: &PairTraversal,
667    apply: Apply,
668    exec: Exec<'_>,
669) where
670    D: Copy + Send + Sync,
671    S: Copy + Send + Sync,
672    Apply: Fn(&mut D, S) + Sync,
673{
674    let src_ptr = src.data().as_ptr();
675    let src_base = src.offset();
676    let dst_base = dest.offset();
677    let dst_ptr = dest.data_mut().as_mut_ptr();
678    // SAFETY: forwarded caller contract: every logical index of `traversal`
679    // maps to in-bounds slots of both views, injectivity holds whenever the
680    // token can fan out, and the borrows do not overlap.
681    unsafe {
682        match traversal {
683            PairTraversal::Inline(layout) => {
684                replay_fused_raw(dst_ptr, dst_base, src_ptr, src_base, layout, apply, exec)
685            }
686            PairTraversal::Spilled(axes) => {
687                replay_spilled_raw(dst_ptr, dst_base, src_ptr, src_base, axes, apply, exec)
688            }
689        }
690    }
691}
692
693/// Pointer-level fused replay shared by [`CopyPlan`], [`UpdatePlan`], the
694/// plan-free `replay_*_raw` entries and the dynamic slice plans, which replay
695/// a fused window layout at a runtime base offset.
696///
697/// Splits `0..total` across the budget of `exec` when the repository threshold
698/// allows more than one slot, otherwise runs the serial kernel directly.
699///
700/// # Safety
701///
702/// Every logical index of `layout`, mapped through its destination strides
703/// from `dst_base` and its source strides from `src_base`, must be an
704/// in-bounds slot of the destination and source allocations, and the product
705/// of its extents must not overflow `usize`. Unless `exec.is_serial()`, the
706/// destination strides must be injective over `layout`; a serial replay
707/// updates a repeated slot once per logical index, in order. The destination
708/// must not overlap the source, and nothing else may access the destination
709/// slots during the call.
710pub(crate) unsafe fn replay_fused_raw<D, S, Apply>(
711    dst_ptr: *mut D,
712    dst_base: isize,
713    src_ptr: *const S,
714    src_base: isize,
715    layout: &FusedPairLayout,
716    apply: Apply,
717    exec: Exec<'_>,
718) where
719    D: Copy + Send + Sync,
720    S: Copy + Send + Sync,
721    Apply: Fn(&mut D, S) + Sync,
722{
723    let total = fused_total(layout);
724    if total == 0 {
725        return;
726    }
727    let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
728    if !exec.is_serial() {
729        let dst_ptr = crate::threading::SendPtr(dst_ptr);
730        let src_ptr = crate::threading::SendPtr(src_ptr as *mut S);
731        crate::threading::parallel_for_each(0..total, exec, &|range| {
732            // SAFETY: the caller contract makes every index in bounds; the
733            // token is not serial, so the destination strides are injective
734            // and the disjoint worker ranges write disjoint slots; the source
735            // is only read.
736            unsafe {
737                apply_fused_range(
738                    dst_ptr.as_ptr(),
739                    dst_base,
740                    src_ptr.as_const(),
741                    src_base,
742                    layout,
743                    range.start,
744                    range.len(),
745                    &apply,
746                );
747            }
748        });
749        return;
750    }
751    // SAFETY: forwarded caller contract over the full range `0..total`; one
752    // sequential walk needs no injectivity.
753    unsafe {
754        apply_fused_range(
755            dst_ptr, dst_base, src_ptr, src_base, layout, 0, total, &apply,
756        );
757    }
758}
759
760/// Pointer-level replay of a spilled axis list.
761///
762/// Mirrors [`replay_fused_raw`] including its worker split; kept separate so
763/// that the inline replay keeps its fixed-rank shape.
764///
765/// # Safety
766///
767/// The [`replay_fused_raw`] contract, for `axes`.
768pub(crate) unsafe fn replay_spilled_raw<D, S, Apply>(
769    dst_ptr: *mut D,
770    dst_base: isize,
771    src_ptr: *const S,
772    src_base: isize,
773    axes: &[FusedAxis],
774    apply: Apply,
775    exec: Exec<'_>,
776) where
777    D: Copy + Send + Sync,
778    S: Copy + Send + Sync,
779    Apply: Fn(&mut D, S) + Sync,
780{
781    let total = axes.iter().map(|axis| axis.dim).product::<usize>();
782    if total == 0 {
783        return;
784    }
785    let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
786    if !exec.is_serial() {
787        let dst_ptr = crate::threading::SendPtr(dst_ptr);
788        let src_ptr = crate::threading::SendPtr(src_ptr as *mut S);
789        crate::threading::parallel_for_each(0..total, exec, &|range| {
790            // SAFETY: as in `replay_fused_raw`: in-bounds indices, injective
791            // destination for a non-serial token, disjoint worker ranges.
792            unsafe {
793                apply_spilled_range(
794                    dst_ptr.as_ptr(),
795                    dst_base,
796                    src_ptr.as_const(),
797                    src_base,
798                    axes,
799                    range.start,
800                    range.len(),
801                    &apply,
802                );
803            }
804        });
805        return;
806    }
807    // SAFETY: forwarded caller contract over the full range `0..total`; one
808    // sequential walk needs no injectivity.
809    unsafe {
810        apply_spilled_range(dst_ptr, dst_base, src_ptr, src_base, axes, 0, total, &apply);
811    }
812}
813
814#[cfg(test)]
815#[path = "copy_plan/tests/tests.rs"]
816mod tests;
817
818#[cfg(test)]
819#[path = "copy_plan/tests/parallel_tests.rs"]
820mod parallel_tests;