strided_basic/copy_plan.rs
1//! Prepared (compile-once, execute-many) copy plans over raw strided layouts.
2//!
3//! [`copy_scale_raw`](crate::copy_scale_raw) and friends rebuild the fused
4//! loop nest on every call; for prepared-replay consumers that issue many
5//! small copies with a fixed layout, that per-call planning dominates
6//! (see issue #139). [`CopyPlan`] splits the work: [`CopyPlan::compile`]
7//! validates the layout pair and builds the fused traversal once,
8//! [`CopyPlan::execute`]/[`CopyPlan::execute_scale`]/[`CopyPlan::execute_conj`]
9//! replay it with no planning and no heap allocation for fused ranks at most
10//! [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
11
12use core::{
13 marker::PhantomData,
14 mem::MaybeUninit,
15 ops::{Add, Mul},
16};
17
18use num_traits::{One, Zero};
19
20use crate::cost::KernelClass;
21use crate::raw_ops::{
22 apply_fused_range, apply_spilled_range, fuse_pair_traversal, fused_total, FusedAxis,
23 FusedPairLayout, PairTraversal,
24};
25use crate::{ElementOpApply, Exec, RawStridedMut, RawStridedRef, Result, StridedError};
26
27// Same pattern as map_view.rs / outer_product.rs: inline storage in both
28// feature configurations. Only `compile` touches these; `execute*` never
29// allocates for them. (A spilled fused layout needs one cursor allocation per
30// replayed range, so an `execute*` can allocate once per worker range above the
31// inline rank bound.)
32type AxisVec<T> = crate::inline::InlineVec<T>;
33
34pub(crate) trait OverwriteWriter<T> {
35 fn dims(&self) -> &[usize];
36 fn strides(&self) -> &[isize];
37 fn offset(&self) -> isize;
38 /// # Safety
39 /// The caller must use the pointer only for the validated allocation and
40 /// logical layout represented by this writer.
41 unsafe fn data_ptr(&mut self) -> *mut T;
42 /// # Safety
43 /// The offset must be an in-bounds logical destination proven by layout.
44 unsafe fn write_at(&mut self, offset: isize, value: T);
45}
46
47pub(crate) trait ReadModifyWrite<T>: OverwriteWriter<T> {
48 /// # Safety
49 /// The offset must be an in-bounds initialized slot covered by the
50 /// traversal's copy and disjointness proof.
51 unsafe fn add_at<F>(&mut self, offset: isize, value: T, combine: F)
52 where
53 F: FnOnce(T, T) -> T;
54}
55
56impl<'a, T> OverwriteWriter<T> for RawStridedMut<'a, T> {
57 fn dims(&self) -> &[usize] {
58 self.dims()
59 }
60 fn strides(&self) -> &[isize] {
61 self.strides()
62 }
63 fn offset(&self) -> isize {
64 self.offset()
65 }
66 unsafe fn data_ptr(&mut self) -> *mut T {
67 self.data_mut().as_mut_ptr()
68 }
69 unsafe fn write_at(&mut self, offset: isize, value: T) {
70 // SAFETY: the prepared layout validates every logical destination.
71 unsafe { self.data_mut().as_mut_ptr().offset(offset).write(value) }
72 }
73}
74
75impl<'a, T> ReadModifyWrite<T> for RawStridedMut<'a, T>
76where
77 T: Add<Output = T>,
78{
79 #[inline(always)]
80 unsafe fn add_at<F>(&mut self, offset: isize, value: T, combine: F)
81 where
82 F: FnOnce(T, T) -> T,
83 {
84 // SAFETY: the copy or initialized caller proves this logical slot.
85 unsafe {
86 let ptr = self.data_mut().as_mut_ptr().offset(offset);
87 ptr.write(combine(ptr.read(), value));
88 }
89 }
90}
91
92impl<'a, T> OverwriteWriter<T> for RawStridedMut<'a, MaybeUninit<T>> {
93 fn dims(&self) -> &[usize] {
94 self.dims()
95 }
96 fn strides(&self) -> &[isize] {
97 self.strides()
98 }
99 fn offset(&self) -> isize {
100 self.offset()
101 }
102 unsafe fn data_ptr(&mut self) -> *mut T {
103 self.data_mut().as_mut_ptr().cast()
104 }
105 unsafe fn write_at(&mut self, offset: isize, value: T) {
106 // SAFETY: the prepared layout validates every logical destination.
107 unsafe {
108 self.data_mut()
109 .as_mut_ptr()
110 .offset(offset)
111 .write(MaybeUninit::new(value))
112 }
113 }
114}
115
116pub(crate) struct InitializedRawDest<'a, T> {
117 ptr: *mut T,
118 extent: usize,
119 dims: &'a [usize],
120 strides: &'a [isize],
121 offset: isize,
122 _marker: PhantomData<&'a mut [MaybeUninit<T>]>,
123}
124
125impl<'a, T> OverwriteWriter<T> for InitializedRawDest<'a, T> {
126 fn dims(&self) -> &[usize] {
127 self.dims
128 }
129 fn strides(&self) -> &[isize] {
130 self.strides
131 }
132 fn offset(&self) -> isize {
133 self.offset
134 }
135 unsafe fn data_ptr(&mut self) -> *mut T {
136 self.ptr
137 }
138 unsafe fn write_at(&mut self, offset: isize, value: T) {
139 debug_assert!(offset >= 0 && (offset as usize) < self.extent);
140 // SAFETY: the copy proof and extent check cover this logical slot.
141 unsafe { self.ptr.offset(offset).write(value) }
142 }
143}
144
145impl<'a, T> ReadModifyWrite<T> for InitializedRawDest<'a, T>
146where
147 T: Add<Output = T>,
148{
149 #[inline(always)]
150 unsafe fn add_at<F>(&mut self, offset: isize, value: T, combine: F)
151 where
152 F: FnOnce(T, T) -> T,
153 {
154 debug_assert!(offset >= 0 && (offset as usize) < self.extent);
155 // SAFETY: the copy proof and extent check cover this logical slot.
156 unsafe {
157 let ptr = self.ptr.offset(offset);
158 ptr.write(combine(ptr.read(), value));
159 }
160 }
161}
162
163/// A compiled copy traversal for one `(dims, dst_strides, src_strides)`
164/// layout pair.
165///
166/// `compile` proves the layout facts once (rank agreement, extent overflow,
167/// destination injectivity) and fuses/orders the loop nest; each `execute*`
168/// call then only re-checks the per-call facts (that the supplied views carry
169/// exactly the compiled layout) before replaying the prepared loops.
170///
171/// Overlapping `src`/`dest` memory is not supported, matching the rest of the
172/// crate.
173///
174/// Ranks above [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT) are supported by the same fused
175/// traversal, which then stores its axes in one rank-bounded allocation.
176///
177/// # Example
178///
179/// ```rust
180/// use strided_basic::{CopyPlan, Exec, RawStridedMut, RawStridedRef};
181///
182/// let dims = [2usize, 3];
183/// let src_strides = [3isize, 1];
184/// let dst_strides = [1isize, 2]; // transposed destination
185/// let plan = CopyPlan::compile(&dims, &dst_strides, &src_strides).unwrap();
186///
187/// let src = [0.0f64, 1.0, 2.0, 10.0, 11.0, 12.0];
188/// let mut dst = [0.0f64; 6];
189/// let src_ref = RawStridedRef::new(&src, &dims, &src_strides, 0).unwrap();
190/// let mut dst_mut = RawStridedMut::new(&mut dst, &dims, &dst_strides, 0).unwrap();
191/// plan.execute(&mut dst_mut, &src_ref, Exec::serial()).unwrap();
192/// assert_eq!(dst, [0.0, 10.0, 1.0, 11.0, 2.0, 12.0]);
193/// ```
194#[derive(Clone, Debug, Default)]
195pub struct CopyPlan {
196 dims: AxisVec<usize>,
197 dst_strides: AxisVec<isize>,
198 src_strides: AxisVec<isize>,
199 fused: PairTraversal,
200}
201
202impl CopyPlan {
203 /// `None` when the fused traversal is stored in the rank-bounded spill.
204 #[inline]
205 pub(crate) fn fused_layout(&self) -> Option<&FusedPairLayout> {
206 self.fused.inline()
207 }
208
209 /// Copy into uninitialized storage, then lend the initialized destination to `f`.
210 ///
211 /// [`Self::execute_uninit`] initializes every reachable logical destination
212 /// element before `f` runs, so the [`InitializedRawDest`] receipt handed to `f`
213 /// may read and overwrite exactly those offsets. Slots outside the plan's
214 /// reachable set are not written by that copy and it makes no claim about
215 /// them: a caller that handed in a `MaybeUninit` array still owns those slots,
216 /// so they keep whatever initialization the caller left. With
217 /// `CopyPlan::compile(&[2], &[2], &[1])` the offset-1 hole keeps the caller's
218 /// value (measured `[7.0, 99.0, 8.0]` for `src = [7, 8]`) rather than
219 /// becoming uninitialized. That is why the receipt exposes
220 /// `write_at`/`add_at` (copy_plan.rs:138, :150) instead of a typed slice: their
221 /// contracts require an offset proven by this plan's layout. The receipt borrows
222 /// the destination for `f`'s lifetime, so a read-modify-write consumer (the
223 /// gather and scatter update replay in gather_plan.rs) sees the copied operand
224 /// once and cannot alias the source.
225 ///
226 /// # Errors
227 ///
228 /// Returns the [`Self::execute_uninit`] errors (`PlanLayoutMismatch`, before any
229 /// write); `f` runs only on success.
230 pub(crate) fn execute_uninit_then<'a, T, R>(
231 &self,
232 dest: &'a mut RawStridedMut<'a, MaybeUninit<T>>,
233 src: &RawStridedRef<'_, T>,
234 mut exec: Exec<'_>,
235 f: impl for<'b, 'e> FnOnce(InitializedRawDest<'b, T>, Exec<'e>) -> R,
236 ) -> Result<R>
237 where
238 T: Copy + Send + Sync,
239 {
240 self.execute_uninit(dest, src, exec.reborrow())?;
241 let data = dest.data_mut();
242 let receipt = InitializedRawDest {
243 ptr: data.as_mut_ptr().cast(),
244 extent: data.len(),
245 dims: dest.dims(),
246 strides: dest.strides(),
247 offset: dest.offset(),
248 _marker: PhantomData,
249 };
250 Ok(f(receipt, exec))
251 }
252
253 /// Compile a copy plan for the given layout pair.
254 ///
255 /// Performs the layout validation and traversal construction
256 /// (fuse + order) once.
257 ///
258 /// # Errors
259 ///
260 /// Returns [`StridedError::StrideLengthMismatch`] when `dims`,
261 /// `dst_strides`, and `src_strides` do not all have the same length,
262 /// [`StridedError::OffsetOverflow`] when the total element count does not
263 /// fit `usize`, and [`StridedError::NonInjectiveOutputLayout`] when the
264 /// destination layout cannot be proven injective (aliasing elements, or a
265 /// span the check cannot represent).
266 pub fn compile(dims: &[usize], dst_strides: &[isize], src_strides: &[isize]) -> Result<Self> {
267 if dims.len() != dst_strides.len() || dims.len() != src_strides.len() {
268 return Err(StridedError::StrideLengthMismatch);
269 }
270 // A zero-sized layout has zero elements even when its other extents
271 // overflow; only a nonempty overflowing count is rejected.
272 crate::kernel::total_len(dims)?;
273 if !crate::layout_check::is_injective_layout(dims, dst_strides) {
274 return Err(StridedError::NonInjectiveOutputLayout);
275 }
276 Ok(Self {
277 dims: dims.into(),
278 dst_strides: dst_strides.into(),
279 src_strides: src_strides.into(),
280 fused: fuse_pair_traversal(dims, dst_strides, src_strides),
281 })
282 }
283
284 /// Check the per-call facts: the supplied views must carry exactly the
285 /// compiled layout. Buffer bounds against that layout were already proven
286 /// by [`RawStridedRef::new`]/[`RawStridedMut::new`] (or asserted by the
287 /// caller of the `new_unchecked` constructors), so layout equality is the
288 /// complete precondition for the pointer-based fused replay below.
289 fn check_call<D, S>(
290 &self,
291 dest: &RawStridedMut<'_, D>,
292 src: &RawStridedRef<'_, S>,
293 ) -> Result<()> {
294 check_layout_pair(&self.dims, &self.dst_strides, &self.src_strides, dest, src)
295 }
296
297 /// Check the per-call facts, then replay the compiled traversal.
298 fn replay<D, S, Apply>(
299 &self,
300 dest: &mut RawStridedMut<'_, D>,
301 src: &RawStridedRef<'_, S>,
302 apply: Apply,
303 exec: Exec<'_>,
304 ) -> Result<()>
305 where
306 D: Copy + Send + Sync,
307 S: Copy + Send + Sync,
308 Apply: Fn(&mut D, S) + Sync,
309 {
310 self.check_call(dest, src)?;
311 // SAFETY: `check_call` proved both views carry exactly the compiled
312 // layout, and `fused` is the fusion of that layout (only `compile`
313 // and `Default`, the rank-zero layout, build it). `compile` checked
314 // the element count and proved the destination injective.
315 unsafe { replay_fused(dest, src, &self.fused, apply, exec) };
316 Ok(())
317 }
318
319 /// `dest = src` into a potentially uninitialized destination.
320 ///
321 /// On success every reachable logical destination element is initialized.
322 /// Non-reachable holes in the backing allocation are not written.
323 ///
324 /// # Errors
325 ///
326 /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
327 /// carry the compiled layout, before any write.
328 pub fn execute_uninit<T>(
329 &self,
330 dest: &mut RawStridedMut<'_, MaybeUninit<T>>,
331 src: &RawStridedRef<'_, T>,
332 exec: Exec<'_>,
333 ) -> Result<()>
334 where
335 T: Copy + Send + Sync,
336 {
337 self.replay(
338 dest,
339 src,
340 |dst: &mut MaybeUninit<T>, value: T| {
341 dst.write(value);
342 },
343 exec,
344 )
345 }
346
347 /// `dest = src`. Allocation-free for fused ranks at most
348 /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
349 ///
350 /// # Errors
351 ///
352 /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
353 /// carry the compiled layout, before any write.
354 pub fn execute<T>(
355 &self,
356 dest: &mut RawStridedMut<'_, T>,
357 src: &RawStridedRef<'_, T>,
358 exec: Exec<'_>,
359 ) -> Result<()>
360 where
361 T: Copy + Send + Sync,
362 {
363 self.replay(dest, src, |dst: &mut T, value: T| *dst = value, exec)
364 }
365
366 /// `dest = scale * src`. Allocation-free for fused ranks at most
367 /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
368 ///
369 /// # Errors
370 ///
371 /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
372 /// carry the compiled layout, before any write.
373 pub fn execute_scale<T>(
374 &self,
375 dest: &mut RawStridedMut<'_, T>,
376 src: &RawStridedRef<'_, T>,
377 scale: T,
378 exec: Exec<'_>,
379 ) -> Result<()>
380 where
381 T: Copy + Mul<T, Output = T> + Send + Sync,
382 {
383 self.replay(
384 dest,
385 src,
386 |dst: &mut T, value: T| *dst = scale * value,
387 exec,
388 )
389 }
390
391 /// `dest = conj(src)`. Allocation-free for fused ranks at most
392 /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
393 ///
394 /// # Errors
395 ///
396 /// Returns [`StridedError::PlanLayoutMismatch`] when the views do not
397 /// carry the compiled layout, before any write.
398 pub fn execute_conj<T>(
399 &self,
400 dest: &mut RawStridedMut<'_, T>,
401 src: &RawStridedRef<'_, T>,
402 exec: Exec<'_>,
403 ) -> Result<()>
404 where
405 T: Copy + ElementOpApply + Send + Sync,
406 {
407 self.replay(dest, src, |dst: &mut T, value: T| *dst = value.conj(), exec)
408 }
409}
410
411/// Verify that a pair of raw views carries exactly a compiled layout.
412///
413/// Shared by [`CopyPlan`] and [`UpdatePlan`], whose `compile` methods prove the
414/// layout facts once. Buffer bounds against that layout were already proven by
415/// [`RawStridedRef::new`]/[`RawStridedMut::new`] (or asserted by the caller of
416/// the `new_unchecked` constructors), so layout equality is the complete
417/// per-call precondition for the pointer-based fused replay.
418fn check_layout_pair<D, S>(
419 dims: &[usize],
420 dst_strides: &[isize],
421 src_strides: &[isize],
422 dest: &RawStridedMut<'_, D>,
423 src: &RawStridedRef<'_, S>,
424) -> Result<()> {
425 if dest.dims() != dims
426 || src.dims() != dims
427 || dest.strides() != dst_strides
428 || src.strides() != src_strides
429 {
430 return Err(StridedError::PlanLayoutMismatch);
431 }
432 Ok(())
433}
434
435/// A compiled read-modify-write traversal for one `(dims, dst_strides,
436/// src_strides)` layout pair.
437///
438/// [`execute`](Self::execute) computes `dest = alpha * f(src) + beta * dest` by
439/// replaying the same fused pair traversal as [`CopyPlan`]: `compile` proves
440/// the layout facts once (rank agreement, extent overflow, destination
441/// injectivity) and each call only re-checks that the supplied views carry the
442/// compiled layout before running monomorphized loops. Execute is
443/// allocation-free for fused ranks at most
444/// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT), and the injective
445/// destination makes a large replay parallel-safe exactly as in [`CopyPlan`].
446///
447/// Overlapping `src`/`dest` memory is not supported, matching the rest of the
448/// crate.
449///
450/// # Example
451///
452/// ```rust
453/// use strided_basic::{Exec, RawStridedMut, RawStridedRef, UpdatePlan};
454///
455/// let dims = [2usize, 2];
456/// let dst_strides = [1isize, 2];
457/// let src_strides = [1isize, 2];
458/// let plan = UpdatePlan::compile(&dims, &dst_strides, &src_strides).unwrap();
459///
460/// let src = [1.0f64, 2.0, 3.0, 4.0];
461/// let mut dst = [10.0f64, 20.0, 30.0, 40.0];
462/// let src_ref = RawStridedRef::new(&src, &dims, &src_strides, 0).unwrap();
463/// let mut dst_mut = RawStridedMut::new(&mut dst, &dims, &dst_strides, 0).unwrap();
464/// // dst = 2 * src + 3 * dst
465/// plan.execute(&mut dst_mut, &src_ref, 2.0, 3.0, |x| x, Exec::serial()).unwrap();
466/// assert_eq!(dst, [32.0, 64.0, 96.0, 128.0]);
467/// ```
468#[derive(Clone, Debug)]
469pub struct UpdatePlan {
470 dims: AxisVec<usize>,
471 dst_strides: AxisVec<isize>,
472 src_strides: AxisVec<isize>,
473 fused: PairTraversal,
474}
475
476impl UpdatePlan {
477 /// Compile an update plan for the given layout pair.
478 ///
479 /// Performs the same layout validation as [`CopyPlan::compile`].
480 ///
481 /// # Errors
482 ///
483 /// Returns [`StridedError::StrideLengthMismatch`] when `dims`,
484 /// `dst_strides`, and `src_strides` do not all have the same length,
485 /// [`StridedError::OffsetOverflow`] when the total element count does not
486 /// fit `usize`, and [`StridedError::NonInjectiveOutputLayout`] when the
487 /// destination layout cannot be proven injective: two logical indices name
488 /// one slot (so a parallel replay would write that slot once per index), or
489 /// the offset span cannot be represented, leaving injectivity undecided.
490 pub fn compile(dims: &[usize], dst_strides: &[isize], src_strides: &[isize]) -> Result<Self> {
491 if dims.len() != dst_strides.len() || dims.len() != src_strides.len() {
492 return Err(StridedError::StrideLengthMismatch);
493 }
494 crate::kernel::total_len(dims)?;
495 if !crate::layout_check::is_injective_layout(dims, dst_strides) {
496 return Err(StridedError::NonInjectiveOutputLayout);
497 }
498 Ok(Self {
499 dims: dims.into(),
500 dst_strides: dst_strides.into(),
501 src_strides: src_strides.into(),
502 fused: fuse_pair_traversal(dims, dst_strides, src_strides),
503 })
504 }
505
506 /// Check the per-call facts, then replay the compiled traversal.
507 fn replay<D, S, Apply>(
508 &self,
509 dest: &mut RawStridedMut<'_, D>,
510 src: &RawStridedRef<'_, S>,
511 apply: Apply,
512 exec: Exec<'_>,
513 ) -> Result<()>
514 where
515 D: Copy + Send + Sync,
516 S: Copy + Send + Sync,
517 Apply: Fn(&mut D, S) + Sync,
518 {
519 check_layout_pair(&self.dims, &self.dst_strides, &self.src_strides, dest, src)?;
520 // SAFETY: as in `CopyPlan::replay`: the views carry the compiled
521 // layout, `fused` is its fusion (built only by `compile`), and
522 // `compile` checked the count and destination injectivity.
523 unsafe { replay_fused(dest, src, &self.fused, apply, exec) };
524 Ok(())
525 }
526
527 /// `dest = alpha * f(src) + beta * dest`.
528 ///
529 /// `f` is applied to the source element before scaling. When `beta` is
530 /// zero (`-0.0` included; `NaN` is not zero) the replay is write-only: the
531 /// old destination element is never read, so a NaN-filled or uninitialized
532 /// destination cannot contaminate the result, matching BLAS `beta = 0`
533 /// semantics. Otherwise the old destination element is read through the
534 /// destination layout. Allocation-free for fused ranks at most
535 /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
536 ///
537 /// # Errors
538 ///
539 /// [`StridedError::PlanLayoutMismatch`] when the supplied views do not
540 /// carry the compiled layout, before any write.
541 pub fn execute<T, F>(
542 &self,
543 dest: &mut RawStridedMut<'_, T>,
544 src: &RawStridedRef<'_, T>,
545 alpha: T,
546 beta: T,
547 f: F,
548 exec: Exec<'_>,
549 ) -> Result<()>
550 where
551 T: Copy + Send + Sync + Add<Output = T> + Mul<Output = T> + Zero + One + PartialEq,
552 F: Fn(T) -> T + Send + Sync,
553 {
554 if beta.is_zero() {
555 // Write-only: never form a read of the old destination.
556 if alpha.is_one() {
557 self.replay(dest, src, |dst: &mut T, value: T| *dst = f(value), exec)
558 } else {
559 self.replay(
560 dest,
561 src,
562 |dst: &mut T, value: T| *dst = alpha * f(value),
563 exec,
564 )
565 }
566 } else {
567 self.replay(
568 dest,
569 src,
570 |dst: &mut T, value: T| *dst = alpha * f(value) + beta * *dst,
571 exec,
572 )
573 }
574 }
575
576 /// `dest = alpha * f(src)` into a potentially uninitialized destination.
577 ///
578 /// This is [`execute`](Self::execute) with `beta = 0`: the destination is
579 /// never read. On success every reachable logical destination element is
580 /// initialized; non-reachable holes in the backing allocation are not
581 /// written. Allocation-free for fused ranks at most
582 /// [`RAW_FUSED_RANK_LIMIT`](crate::RAW_FUSED_RANK_LIMIT).
583 ///
584 /// # Errors
585 ///
586 /// [`StridedError::PlanLayoutMismatch`] when the supplied views do not
587 /// carry the compiled layout, before any write.
588 ///
589 /// # Examples
590 ///
591 /// ```rust
592 /// use core::mem::MaybeUninit;
593 /// use strided_basic::{Exec, RawStridedMut, RawStridedRef, UpdatePlan};
594 ///
595 /// let dims = [2usize, 2];
596 /// let strides = [1isize, 2];
597 /// let plan = UpdatePlan::compile(&dims, &strides, &strides).unwrap();
598 /// let src = [1.0f64, 2.0, 3.0, 4.0];
599 /// let mut storage = [MaybeUninit::<f64>::uninit(); 4];
600 /// let mut dst = RawStridedMut::new(&mut storage, &dims, &strides, 0).unwrap();
601 /// let source = RawStridedRef::new(&src, &dims, &strides, 0).unwrap();
602 /// // dst = 2 * src
603 /// plan.execute_uninit(&mut dst, &source, 2.0, |x| x, Exec::serial()).unwrap();
604 /// drop(dst);
605 /// // SAFETY: every reachable destination slot was initialized above.
606 /// let values: Vec<f64> = storage.iter().map(|v| unsafe { v.assume_init() }).collect();
607 /// assert_eq!(values, [2.0, 4.0, 6.0, 8.0]);
608 /// ```
609 pub fn execute_uninit<T, F>(
610 &self,
611 dest: &mut RawStridedMut<'_, MaybeUninit<T>>,
612 src: &RawStridedRef<'_, T>,
613 alpha: T,
614 f: F,
615 exec: Exec<'_>,
616 ) -> Result<()>
617 where
618 T: Copy + Send + Sync + Mul<Output = T> + One + PartialEq,
619 F: Fn(T) -> T + Send + Sync,
620 {
621 if alpha.is_one() {
622 self.replay(
623 dest,
624 src,
625 |dst: &mut MaybeUninit<T>, value: T| {
626 dst.write(f(value));
627 },
628 exec,
629 )
630 } else {
631 self.replay(
632 dest,
633 src,
634 |dst: &mut MaybeUninit<T>, value: T| {
635 dst.write(alpha * f(value));
636 },
637 exec,
638 )
639 }
640 }
641}
642
643/// Replay `traversal` over a pair of raw views, splitting it across the budget
644/// of `exec` when the repository threshold allows it.
645///
646/// The logical index space `0..total` (column-major over the traversal axes)
647/// is split into contiguous worker ranges, which divides the outer axes and,
648/// when there are fewer outer positions than workers, also chunks long inner
649/// runs (for example a rank-1 contiguous copy).
650///
651/// # Safety
652///
653/// - `traversal` must replay exactly the logical index space of the two
654/// views: either their dims and strides fused by `fuse_pair_traversal`, or
655/// built from them by `PairTraversal::given`. The views' own bounds proofs
656/// ([`RawStridedRef::new`], [`RawStridedMut::new`], or the caller of their
657/// `new_unchecked`) then make every replayed offset in-bounds.
658/// - The product of the traversal extents must not overflow `usize` (a
659/// checked count, zero included).
660/// - Unless `exec.is_serial()`, the destination strides must be injective
661/// over the traversal.
662/// - The destination and source must not overlap.
663pub(crate) unsafe fn replay_fused<D, S, Apply>(
664 dest: &mut RawStridedMut<'_, D>,
665 src: &RawStridedRef<'_, S>,
666 traversal: &PairTraversal,
667 apply: Apply,
668 exec: Exec<'_>,
669) where
670 D: Copy + Send + Sync,
671 S: Copy + Send + Sync,
672 Apply: Fn(&mut D, S) + Sync,
673{
674 let src_ptr = src.data().as_ptr();
675 let src_base = src.offset();
676 let dst_base = dest.offset();
677 let dst_ptr = dest.data_mut().as_mut_ptr();
678 // SAFETY: forwarded caller contract: every logical index of `traversal`
679 // maps to in-bounds slots of both views, injectivity holds whenever the
680 // token can fan out, and the borrows do not overlap.
681 unsafe {
682 match traversal {
683 PairTraversal::Inline(layout) => {
684 replay_fused_raw(dst_ptr, dst_base, src_ptr, src_base, layout, apply, exec)
685 }
686 PairTraversal::Spilled(axes) => {
687 replay_spilled_raw(dst_ptr, dst_base, src_ptr, src_base, axes, apply, exec)
688 }
689 }
690 }
691}
692
693/// Pointer-level fused replay shared by [`CopyPlan`], [`UpdatePlan`], the
694/// plan-free `replay_*_raw` entries and the dynamic slice plans, which replay
695/// a fused window layout at a runtime base offset.
696///
697/// Splits `0..total` across the budget of `exec` when the repository threshold
698/// allows more than one slot, otherwise runs the serial kernel directly.
699///
700/// # Safety
701///
702/// Every logical index of `layout`, mapped through its destination strides
703/// from `dst_base` and its source strides from `src_base`, must be an
704/// in-bounds slot of the destination and source allocations, and the product
705/// of its extents must not overflow `usize`. Unless `exec.is_serial()`, the
706/// destination strides must be injective over `layout`; a serial replay
707/// updates a repeated slot once per logical index, in order. The destination
708/// must not overlap the source, and nothing else may access the destination
709/// slots during the call.
710pub(crate) unsafe fn replay_fused_raw<D, S, Apply>(
711 dst_ptr: *mut D,
712 dst_base: isize,
713 src_ptr: *const S,
714 src_base: isize,
715 layout: &FusedPairLayout,
716 apply: Apply,
717 exec: Exec<'_>,
718) where
719 D: Copy + Send + Sync,
720 S: Copy + Send + Sync,
721 Apply: Fn(&mut D, S) + Sync,
722{
723 let total = fused_total(layout);
724 if total == 0 {
725 return;
726 }
727 let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
728 if !exec.is_serial() {
729 let dst_ptr = crate::threading::SendPtr(dst_ptr);
730 let src_ptr = crate::threading::SendPtr(src_ptr as *mut S);
731 crate::threading::parallel_for_each(0..total, exec, &|range| {
732 // SAFETY: the caller contract makes every index in bounds; the
733 // token is not serial, so the destination strides are injective
734 // and the disjoint worker ranges write disjoint slots; the source
735 // is only read.
736 unsafe {
737 apply_fused_range(
738 dst_ptr.as_ptr(),
739 dst_base,
740 src_ptr.as_const(),
741 src_base,
742 layout,
743 range.start,
744 range.len(),
745 &apply,
746 );
747 }
748 });
749 return;
750 }
751 // SAFETY: forwarded caller contract over the full range `0..total`; one
752 // sequential walk needs no injectivity.
753 unsafe {
754 apply_fused_range(
755 dst_ptr, dst_base, src_ptr, src_base, layout, 0, total, &apply,
756 );
757 }
758}
759
760/// Pointer-level replay of a spilled axis list.
761///
762/// Mirrors [`replay_fused_raw`] including its worker split; kept separate so
763/// that the inline replay keeps its fixed-rank shape.
764///
765/// # Safety
766///
767/// The [`replay_fused_raw`] contract, for `axes`.
768pub(crate) unsafe fn replay_spilled_raw<D, S, Apply>(
769 dst_ptr: *mut D,
770 dst_base: isize,
771 src_ptr: *const S,
772 src_base: isize,
773 axes: &[FusedAxis],
774 apply: Apply,
775 exec: Exec<'_>,
776) where
777 D: Copy + Send + Sync,
778 S: Copy + Send + Sync,
779 Apply: Fn(&mut D, S) + Sync,
780{
781 let total = axes.iter().map(|axis| axis.dim).product::<usize>();
782 if total == 0 {
783 return;
784 }
785 let exec = crate::cost::for_len(exec, KernelClass::Copy, total);
786 if !exec.is_serial() {
787 let dst_ptr = crate::threading::SendPtr(dst_ptr);
788 let src_ptr = crate::threading::SendPtr(src_ptr as *mut S);
789 crate::threading::parallel_for_each(0..total, exec, &|range| {
790 // SAFETY: as in `replay_fused_raw`: in-bounds indices, injective
791 // destination for a non-serial token, disjoint worker ranges.
792 unsafe {
793 apply_spilled_range(
794 dst_ptr.as_ptr(),
795 dst_base,
796 src_ptr.as_const(),
797 src_base,
798 axes,
799 range.start,
800 range.len(),
801 &apply,
802 );
803 }
804 });
805 return;
806 }
807 // SAFETY: forwarded caller contract over the full range `0..total`; one
808 // sequential walk needs no injectivity.
809 unsafe {
810 apply_spilled_range(dst_ptr, dst_base, src_ptr, src_base, axes, 0, total, &apply);
811 }
812}
813
814#[cfg(test)]
815#[path = "copy_plan/tests/tests.rs"]
816mod tests;
817
818#[cfg(test)]
819#[path = "copy_plan/tests/parallel_tests.rs"]
820mod parallel_tests;