Skip to main content

strided_basic/
execution.rs

1//! Execution contract for operation-family implementations.
2//!
3//! This module is the shared kernel-extension interface, not a public mirror
4//! of implementation modules. Prevalidated entries require the caller to prove
5//! the exact destination geometry and every omitted shape check. A layout marker
6//! alone does not prove those obligations. Ordinary callers should use the
7//! checked APIs at the crate root. Every family that can fan out takes the same
8//! [`Exec`] placement token as its last parameter.
9//!
10//! # Examples
11//!
12//! ```
13//! use strided_basic::{Exec, StridedArray, execution::*};
14//! let src = StridedArray::<f64>::col_major(&[2]);
15//! let mut dst = StridedArray::<f64>::col_major(&[2]);
16//! let marker = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
17//! // SAFETY: both arrays have shape [2], distinct storage, and the checked layout.
18//! unsafe { map_into_validated(&mut dst.view_mut(), &src.view(), |x| x, marker, Exec::serial()) }.unwrap();
19//! ```
20//!
21//! A marker alone does not make the prevalidated entry safe to call, which is
22//! why every prevalidated entry is `unsafe`: the compiler rejects a call that
23//! does not acknowledge the caller's obligation.
24//!
25//! ```compile_fail,E0133
26//! use strided_basic::{Exec, StridedArray, execution::*};
27//! let src = StridedArray::<f64>::col_major(&[2]);
28//! let mut dst = StridedArray::<f64>::col_major(&[2]);
29//! let marker = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
30//! map_into_validated(&mut dst.view_mut(), &src.view(), |x| x, marker, Exec::serial()).unwrap();
31//! ```
32pub use crate::cost::{parallel_width, tree_width, KernelClass};
33pub use crate::erased_common::{
34    check_dtype, check_static_indexing_dtype, erased_view, validate_uninit_no_overlap,
35};
36pub use crate::kernel::{ensure_same_shape, KernelPlan, SMALL_TENSOR_THRESHOLD};
37pub use crate::layout_check::is_injective_layout;
38pub use crate::map_view::{
39    validate_destination_layout, validate_destination_layout_without_alloc,
40    ValidatedDestinationLayout,
41};
42pub use crate::threading::{parallel_map_reduce, SendPtr};
43
44use crate::{ElementOp, Exec, InlineVec, OperandVec, Result, StridedView, StridedViewMut};
45
46/// Prevalidated build plan fused for kernel-family implementations.
47///
48/// # Safety
49/// All rank-indexed arrays must have matching lengths and a valid destination
50/// index (when present). Shape products, stride magnitudes, cost arithmetic,
51/// and every intermediate offset used by planning/iteration must be representable.
52/// Iteration blocks must be positive and offsets must correspond to the supplied
53/// layouts; threaded partitioning additionally requires positive
54/// costs. Callback memory accesses must be valid for each generated
55/// block, with disjoint mutable regions when partitions execute concurrently.
56///
57/// # Examples
58///
59/// ```
60/// use strided_basic::execution::*;
61/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
62/// let (dims, _, plan) = unsafe { build_plan_fused(&[2, 3], &[&[1, 2]], Some(0), 8) };
63/// assert_eq!(dims.iter().product::<usize>(), 6);
64/// assert_eq!(plan.block.len(), dims.len());
65/// ```
66#[inline]
67pub unsafe fn build_plan_fused(
68    dims: &[usize],
69    strides_list: &[&[isize]],
70    dest_index: Option<usize>,
71    elem_size: usize,
72) -> (InlineVec<usize>, OperandVec<InlineVec<isize>>, KernelPlan) {
73    // SAFETY: the caller supplies the same invariants as the owning checked entry.
74    crate::kernel::build_plan_fused(dims, strides_list, dest_index, elem_size)
75}
76
77/// Prevalidated build plan fused small for kernel-family implementations.
78///
79/// # Safety
80/// All rank-indexed arrays must have matching lengths and a valid destination
81/// index (when present). Shape products, stride magnitudes, cost arithmetic,
82/// and every intermediate offset used by planning/iteration must be representable.
83/// Iteration blocks must be positive and offsets must correspond to the supplied
84/// layouts; threaded partitioning additionally requires positive
85/// costs. Callback memory accesses must be valid for each generated
86/// block, with disjoint mutable regions when partitions execute concurrently.
87///
88/// # Examples
89///
90/// ```
91/// use strided_basic::execution::*;
92/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
93/// let (dims, _, _) = unsafe { build_plan_fused_small(&[2, 3], &[&[1, 2]]) };
94/// assert_eq!(dims.iter().product::<usize>(), 6);
95/// ```
96#[inline]
97pub unsafe fn build_plan_fused_small(
98    dims: &[usize],
99    strides_list: &[&[isize]],
100) -> (InlineVec<usize>, OperandVec<InlineVec<isize>>, KernelPlan) {
101    // SAFETY: the caller supplies the same invariants as the owning checked entry.
102    crate::kernel::build_plan_fused_small(dims, strides_list)
103}
104
105/// Prevalidated for each inner block preordered for kernel-family implementations.
106///
107/// # Safety
108/// All rank-indexed arrays must have matching lengths and a valid destination
109/// index (when present). Shape products, stride magnitudes, cost arithmetic,
110/// and every intermediate offset used by planning/iteration must be representable.
111/// Iteration blocks must be positive and offsets must correspond to the supplied
112/// layouts; threaded partitioning additionally requires positive
113/// costs. Callback memory accesses must be valid for each generated
114/// block, with disjoint mutable regions when partitions execute concurrently.
115///
116/// # Examples
117///
118/// ```
119/// use strided_basic::execution::*;
120/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
121/// let mut count = 0;
122/// unsafe { for_each_inner_block_preordered(&[4], &[4], &[vec![1]], &[0], |_, n, _| { count += n; Ok(()) }) }.unwrap();
123/// assert_eq!(count, 4);
124/// ```
125///
126/// # Errors
127///
128/// Performs no validation of its own, so the only error it can return is the
129/// one `f` returns; the block walk itself cannot fail. Callers must still
130/// satisfy the safety contract before invoking this prevalidated entry.
131#[inline]
132pub unsafe fn for_each_inner_block_preordered<S: AsRef<[isize]>, F>(
133    dims: &[usize],
134    blocks: &[usize],
135    strides: &[S],
136    initial_offsets: &[isize],
137    f: F,
138) -> Result<()>
139where
140    F: FnMut(&[isize], usize, &[isize]) -> Result<()>,
141{
142    // SAFETY: the caller supplies the same invariants as the owning checked entry.
143    crate::kernel::for_each_inner_block_preordered::<S, F>(
144        dims,
145        blocks,
146        strides,
147        initial_offsets,
148        f,
149    )
150}
151
152/// Prevalidated map into validated for kernel-family implementations.
153///
154/// # Safety
155/// All input shapes must match the destination. `validated` must have been
156/// obtained for this destination's current geometry, proving injectivity;
157/// possessing a marker from another layout is not sufficient. View/descriptor
158/// bounds and aliasing contracts must continue to hold throughout replay.
159///
160/// # Examples
161///
162/// ```
163/// use strided_basic::execution::*;
164/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
165/// use strided_basic::StridedArray;
166/// let src = StridedArray::<f64>::from_parts(vec![2.0; 2], &[2], &[1], 0).unwrap();
167/// let mut dst = StridedArray::<f64>::col_major(&[2]);
168/// let checked = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
169/// unsafe { map_into_validated(&mut dst.view_mut(), &src.view(), |a| a, checked, strided_basic::Exec::serial()) }.unwrap();
170/// assert_eq!(dst.get(&[1]), 2.0);
171/// ```
172///
173/// # Errors
174///
175/// Returns `StridedError::RankMismatch` or `StridedError::ShapeMismatch`
176/// when the source view does not have the destination's ranks and extents,
177/// the only checks the entry keeps. The destination layout is not
178/// re-validated: `validated` is taken as proof of injectivity, and a marker
179/// for this geometry also excludes an unrepresentable element count.
180/// Callers must still satisfy the safety contract before invoking this
181/// prevalidated entry.
182#[inline]
183pub unsafe fn map_into_validated<D: Copy + Send + Sync, A: Copy + Send + Sync, Op: ElementOp<A>>(
184    dest: &mut StridedViewMut<D>,
185    src: &StridedView<A, Op>,
186    f: impl crate::MapFn<A, D>,
187    validated: ValidatedDestinationLayout,
188    exec: Exec<'_>,
189) -> Result<()> {
190    // SAFETY: the caller supplies the same invariants as the owning checked entry.
191    crate::map_view::map_into_validated::<D, A, Op>(dest, src, f, validated, exec)
192}
193
194/// Prevalidated zip map2 into validated for kernel-family implementations.
195///
196/// # Safety
197/// All input shapes must match the destination. `validated` must have been
198/// obtained for this destination's current geometry, proving injectivity;
199/// possessing a marker from another layout is not sufficient. View/descriptor
200/// bounds and aliasing contracts must continue to hold throughout replay.
201///
202/// # Examples
203///
204/// ```
205/// use strided_basic::execution::*;
206/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
207/// use strided_basic::StridedArray;
208/// let src = StridedArray::<f64>::from_parts(vec![2.0; 2], &[2], &[1], 0).unwrap();
209/// let mut dst = StridedArray::<f64>::col_major(&[2]);
210/// let checked = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
211/// unsafe { zip_map2_into_validated(&mut dst.view_mut(), &src.view(), &src.view(), |a, b| a + b, checked, strided_basic::Exec::serial()) }.unwrap();
212/// assert_eq!(dst.get(&[1]), 4.0);
213/// ```
214///
215/// # Errors
216///
217/// Performs no shape or destination-layout check of its own and returns
218/// `Ok(())` for every call that satisfies the safety contract: matching
219/// source extents come from that contract, destination injectivity from
220/// `validated`, and a marker for this geometry also excludes an
221/// unrepresentable element count. Callers must still satisfy the safety
222/// contract before invoking this prevalidated entry.
223#[inline]
224pub unsafe fn zip_map2_into_validated<
225    D: Copy + Send + Sync,
226    A: Copy + Send + Sync,
227    B: Copy + Send + Sync,
228    OpA: ElementOp<A>,
229    OpB: ElementOp<B>,
230>(
231    dest: &mut StridedViewMut<D>,
232    a: &StridedView<A, OpA>,
233    b: &StridedView<B, OpB>,
234    f: impl Fn(A, B) -> D + Sync,
235    validated: ValidatedDestinationLayout,
236    exec: Exec<'_>,
237) -> Result<()> {
238    // SAFETY: the caller supplies the same invariants as the owning checked entry.
239    crate::map_view::zip_map2_into_validated::<D, A, B, OpA, OpB>(dest, a, b, f, validated, exec)
240}
241
242/// Prevalidated zip map3 into validated for kernel-family implementations.
243///
244/// # Safety
245/// All input shapes must match the destination. `validated` must have been
246/// obtained for this destination's current geometry, proving injectivity;
247/// possessing a marker from another layout is not sufficient. View/descriptor
248/// bounds and aliasing contracts must continue to hold throughout replay.
249///
250/// # Examples
251///
252/// ```
253/// use strided_basic::execution::*;
254/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
255/// use strided_basic::StridedArray;
256/// let src = StridedArray::<f64>::from_parts(vec![2.0; 2], &[2], &[1], 0).unwrap();
257/// let mut dst = StridedArray::<f64>::col_major(&[2]);
258/// let checked = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
259/// unsafe { zip_map3_into_validated(&mut dst.view_mut(), &src.view(), &src.view(), &src.view(), |a, b, c| a + b + c, checked, strided_basic::Exec::serial()) }.unwrap();
260/// assert_eq!(dst.get(&[1]), 6.0);
261/// ```
262///
263/// # Errors
264///
265/// Returns `StridedError::RankMismatch` or `StridedError::ShapeMismatch`
266/// when any source view does not have the destination's ranks and extents,
267/// the only checks the entry keeps. The destination layout is not
268/// re-validated: `validated` is taken as proof of injectivity, and a marker
269/// for this geometry also excludes an unrepresentable element count.
270/// Callers must still satisfy the safety contract before invoking this
271/// prevalidated entry.
272#[inline]
273pub unsafe fn zip_map3_into_validated<
274    D: Copy + Send + Sync,
275    A: Copy + Send + Sync,
276    B: Copy + Send + Sync,
277    C: Copy + Send + Sync,
278    OpA: ElementOp<A>,
279    OpB: ElementOp<B>,
280    OpC: ElementOp<C>,
281>(
282    dest: &mut StridedViewMut<D>,
283    a: &StridedView<A, OpA>,
284    b: &StridedView<B, OpB>,
285    c: &StridedView<C, OpC>,
286    f: impl Fn(A, B, C) -> D + Sync,
287    validated: ValidatedDestinationLayout,
288    exec: Exec<'_>,
289) -> Result<()> {
290    // SAFETY: the caller supplies the same invariants as the owning checked entry.
291    crate::map_view::zip_map3_into_validated::<D, A, B, C, OpA, OpB, OpC>(
292        dest, a, b, c, f, validated, exec,
293    )
294}
295
296/// Prevalidated zip map4 into validated for kernel-family implementations.
297///
298/// # Safety
299/// All input shapes must match the destination. `validated` must have been
300/// obtained for this destination's current geometry, proving injectivity;
301/// possessing a marker from another layout is not sufficient. View/descriptor
302/// bounds and aliasing contracts must continue to hold throughout replay.
303///
304/// # Examples
305///
306/// ```
307/// use strided_basic::execution::*;
308/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
309/// use strided_basic::StridedArray;
310/// let src = StridedArray::<f64>::from_parts(vec![2.0; 2], &[2], &[1], 0).unwrap();
311/// let mut dst = StridedArray::<f64>::col_major(&[2]);
312/// let checked = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
313/// unsafe { zip_map4_into_validated(&mut dst.view_mut(), &src.view(), &src.view(), &src.view(), &src.view(), |a, b, c, d| a + b + c + d, checked, strided_basic::Exec::serial()) }.unwrap();
314/// assert_eq!(dst.get(&[1]), 8.0);
315/// ```
316///
317/// # Errors
318///
319/// Returns `StridedError::RankMismatch` or `StridedError::ShapeMismatch`
320/// when any source view does not have the destination's ranks and extents,
321/// the only checks the entry keeps. The destination layout is not
322/// re-validated: `validated` is taken as proof of injectivity, and a marker
323/// for this geometry also excludes an unrepresentable element count.
324/// Callers must still satisfy the safety contract before invoking this
325/// prevalidated entry.
326#[inline]
327// Four source views with their element ops, the destination, the closure, the
328// `validated` marker and the token.
329#[allow(clippy::too_many_arguments)]
330pub unsafe fn zip_map4_into_validated<
331    D: Copy + Send + Sync,
332    A: Copy + Send + Sync,
333    B: Copy + Send + Sync,
334    C: Copy + Send + Sync,
335    E: Copy + Send + Sync,
336    OpA: ElementOp<A>,
337    OpB: ElementOp<B>,
338    OpC: ElementOp<C>,
339    OpE: ElementOp<E>,
340>(
341    dest: &mut StridedViewMut<D>,
342    a: &StridedView<A, OpA>,
343    b: &StridedView<B, OpB>,
344    c: &StridedView<C, OpC>,
345    e: &StridedView<E, OpE>,
346    f: impl Fn(A, B, C, E) -> D + Sync,
347    validated: ValidatedDestinationLayout,
348    exec: Exec<'_>,
349) -> Result<()> {
350    // SAFETY: the caller supplies the same invariants as the owning checked entry.
351    crate::map_view::zip_map4_into_validated::<D, A, B, C, E, OpA, OpB, OpC, OpE>(
352        dest, a, b, c, e, f, validated, exec,
353    )
354}
355
356/// Prevalidated map raw into validated for kernel-family implementations.
357///
358/// # Safety
359/// All input shapes must match the destination. `validated` must have been
360/// obtained for this destination's current geometry, proving injectivity;
361/// possessing a marker from another layout is not sufficient. View/descriptor
362/// bounds and aliasing contracts must continue to hold throughout replay.
363///
364/// # Examples
365///
366/// ```
367/// use strided_basic::execution::*;
368/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
369/// use strided_basic::{RawStridedRef, RawStridedMut, Identity};
370/// let values = [2.0_f64; 2];
371/// let src = RawStridedRef::new(&values, &[2], &[1], 0).unwrap();
372/// let mut output = [0.0; 2];
373/// let mut dst = RawStridedMut::new(&mut output, &[2], &[1], 0).unwrap();
374/// let checked = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
375/// unsafe { map_raw_into_validated::<f64, f64, Identity>(&mut dst, &src, |a| a, checked, strided_basic::Exec::serial()) }.unwrap();
376/// assert_eq!(output, [2.0; 2]);
377/// ```
378///
379/// # Errors
380///
381/// Returns `StridedError::RankMismatch` or `StridedError::ShapeMismatch`
382/// when the source does not have the destination's ranks and extents, the
383/// only checks the entry keeps. The destination layout is not re-validated:
384/// `validated` is taken as proof of injectivity, and a marker for this
385/// geometry also excludes an unrepresentable element count. Callers must
386/// still satisfy the safety contract before invoking this prevalidated
387/// entry.
388#[inline]
389pub unsafe fn map_raw_into_validated<
390    D: Copy + Send + Sync,
391    A: Copy + Send + Sync,
392    Op: ElementOp<A>,
393>(
394    dest: &mut crate::RawStridedMut<'_, D>,
395    src: &crate::RawStridedRef<'_, A>,
396    f: impl Fn(A) -> D + Sync,
397    validated: ValidatedDestinationLayout,
398    exec: Exec<'_>,
399) -> Result<()> {
400    // SAFETY: the caller supplies the same invariants as the owning checked entry.
401    crate::map_view::map_raw_into_validated::<D, A, Op>(dest, src, f, validated, exec)
402}
403
404/// Prevalidated zip map2 raw into validated for kernel-family implementations.
405///
406/// # Safety
407/// All input shapes must match the destination. `validated` must have been
408/// obtained for this destination's current geometry, proving injectivity;
409/// possessing a marker from another layout is not sufficient. View/descriptor
410/// bounds and aliasing contracts must continue to hold throughout replay.
411///
412/// # Examples
413///
414/// ```
415/// use strided_basic::execution::*;
416/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
417/// use strided_basic::{RawStridedRef, RawStridedMut, Identity};
418/// let values = [2.0_f64; 2];
419/// let src = RawStridedRef::new(&values, &[2], &[1], 0).unwrap();
420/// let mut output = [0.0; 2];
421/// let mut dst = RawStridedMut::new(&mut output, &[2], &[1], 0).unwrap();
422/// let checked = validate_destination_layout_without_alloc(&[2], &[1]).unwrap();
423/// unsafe { zip_map2_raw_into_validated::<f64, f64, f64, Identity, Identity>(&mut dst, &src, &src, |a, b| a + b, checked, strided_basic::Exec::serial()) }.unwrap();
424/// assert_eq!(output, [4.0; 2]);
425/// ```
426///
427/// # Errors
428///
429/// Returns `StridedError::RankMismatch` or `StridedError::ShapeMismatch`
430/// when either source does not have the destination's ranks and extents,
431/// the only checks the entry keeps. The destination layout is not
432/// re-validated: `validated` is taken as proof of injectivity, and a marker
433/// for this geometry also excludes an unrepresentable element count.
434/// Callers must still satisfy the safety contract before invoking this
435/// prevalidated entry.
436#[inline]
437pub unsafe fn zip_map2_raw_into_validated<
438    D: Copy + Send + Sync,
439    A: Copy + Send + Sync,
440    B: Copy + Send + Sync,
441    OpA: ElementOp<A>,
442    OpB: ElementOp<B>,
443>(
444    dest: &mut crate::RawStridedMut<'_, D>,
445    a: &crate::RawStridedRef<'_, A>,
446    b: &crate::RawStridedRef<'_, B>,
447    f: impl Fn(A, B) -> D + Sync,
448    validated: ValidatedDestinationLayout,
449    exec: Exec<'_>,
450) -> Result<()> {
451    // SAFETY: the caller supplies the same invariants as the owning checked entry.
452    crate::map_view::zip_map2_raw_into_validated::<D, A, B, OpA, OpB>(
453        dest, a, b, f, validated, exec,
454    )
455}
456
457/// Prevalidated compute costs for kernel-family implementations.
458///
459/// # Safety
460/// All rank-indexed arrays must have matching lengths and a valid destination
461/// index (when present). Shape products, stride magnitudes, cost arithmetic,
462/// and every intermediate offset used by planning/iteration must be representable.
463/// Iteration blocks must be positive and offsets must correspond to the supplied
464/// layouts; threaded partitioning additionally requires positive
465/// costs. Callback memory accesses must be valid for each generated
466/// block, with disjoint mutable regions when partitions execute concurrently.
467///
468/// # Examples
469///
470/// ```
471/// use strided_basic::execution::*;
472/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
473/// let costs = unsafe { compute_costs(&[vec![1, 4], vec![1, 8]]) };
474/// assert_eq!(costs.as_slice(), [2, 8]);
475/// ```
476#[inline]
477pub unsafe fn compute_costs<S: AsRef<[isize]>>(all_strides: &[S]) -> InlineVec<isize> {
478    // SAFETY: the caller supplies the same invariants as the owning checked entry.
479    crate::fuse::compute_costs::<S>(all_strides)
480}
481
482/// Prevalidated mapreduce threaded for kernel-family implementations.
483///
484/// `class` is the serial cost class of the leaf (see [`KernelClass`]); every node
485/// of the split runs in parallel only when a fork of its region pays for that
486/// class.
487///
488/// # Safety
489/// All rank-indexed arrays must have matching lengths and a valid destination
490/// index (when present). Shape products, stride magnitudes, cost arithmetic,
491/// and every intermediate offset used by planning/iteration must be representable.
492/// Iteration blocks must be positive and offsets must correspond to the supplied
493/// layouts; threaded partitioning additionally requires positive
494/// costs. Callback memory accesses must be valid for each generated
495/// block, with disjoint mutable regions when partitions execute concurrently.
496///
497/// # Examples
498///
499/// ```
500/// use strided_basic::execution::*;
501/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
502/// use std::sync::atomic::{AtomicUsize, Ordering};
503/// let count = AtomicUsize::new(0);
504/// unsafe { mapreduce_threaded(&[4], &[4], &[vec![1]], &[0], &[2], KernelClass::Map, strided_basic::Exec::serial(), &|dims, _, _, _| { count.fetch_add(dims.iter().product::<usize>(), Ordering::Relaxed); Ok(()) }) }.unwrap();
505/// assert_eq!(count.load(Ordering::Relaxed), 4);
506/// ```
507///
508/// # Errors
509///
510/// Returns `StridedError::OffsetOverflow` when the element count of `dims`
511/// cannot be represented, and otherwise only the error `f` returns; the
512/// splitter itself cannot fail. Callers must still satisfy the safety
513/// contract before invoking this prevalidated entry.
514// Splitter entry: the traversal's dims, blocks, per-operand strides and offsets,
515// the costs, the class, the token and the leaf closure.
516#[allow(clippy::too_many_arguments)]
517#[inline]
518pub unsafe fn mapreduce_threaded<S: AsRef<[isize]>, F>(
519    dims: &[usize],
520    blocks: &[usize],
521    strides_list: &[S],
522    offsets: &[isize],
523    costs: &[isize],
524    class: KernelClass,
525    exec: Exec<'_>,
526    f: &F,
527) -> Result<()>
528where
529    F: Fn(&[usize], &[usize], &[&[isize]], &[isize]) -> Result<()> + Sync,
530{
531    // SAFETY: the caller supplies the same invariants as the owning checked entry.
532    crate::threading::mapreduce_threaded::<S, F>(
533        dims,
534        blocks,
535        strides_list,
536        offsets,
537        costs,
538        class,
539        exec,
540        f,
541    )
542}
543
544/// Full-overwrite indexed replay for an operation-family adapter.
545///
546/// # Safety
547/// The adapter must reject input/output overlap before creating the input
548/// references. All descriptor allocation and initialization contracts must
549/// remain valid for the call. Output slots may be uninitialized; the owning
550/// plan retains its checked layout/index validation and private initialization
551/// receipt. No initialized reference to the output backing may be formed.
552/// Gather into uninitialized output without exposing an initialization receipt.
553///
554/// # Examples
555///
556/// ```
557/// use strided_basic::{RawStridedRef, RawStridedMut};
558/// use strided_basic::execution::*;
559/// use core::mem::MaybeUninit;
560/// use strided_basic::{GatherPlan, GatherSpec};
561/// let spec = GatherSpec { offset_dims: vec![], collapsed_slice_dims: vec![0], start_index_map: vec![0], index_vector_dim: 1, slice_sizes: vec![1] };
562/// let plan = GatherPlan::compile(&[3], &[1], &[2, 1], &[1, 2], &[2], &[1], spec).unwrap();
563/// let src = RawStridedRef::new(&[1_i32, 2, 3], &[3], &[1], 0).unwrap();
564/// let indices = RawStridedRef::new(&[0_i32, 2], &[2, 1], &[1, 2], 0).unwrap();
565/// let mut values = [MaybeUninit::<i32>::uninit(); 2];
566/// let mut dst = RawStridedMut::new(&mut values, &[2], &[1], 0).unwrap();
567/// // SAFETY: input and output allocations are disjoint and layouts match the plan.
568/// unsafe { gather_into_uninit(&plan, &mut dst, &src, &indices, strided_basic::Exec::serial()) }.unwrap();
569/// assert_eq!(unsafe { values[1].assume_init() }, 3);
570/// ```
571///
572/// # Errors
573///
574/// Returns `StridedError::PlanLayoutMismatch` when the destination, operand,
575/// or start-index view does not carry the plan's compiled extents and
576/// strides, before any destination write, and
577/// `StridedError::OffsetOverflow` when a replay offset cannot be
578/// represented. Out-of-range start indices are clamped, not rejected.
579#[inline]
580pub unsafe fn gather_into_uninit<T, I>(
581    plan: &crate::GatherPlan,
582    dest: &mut crate::RawStridedMut<'_, core::mem::MaybeUninit<T>>,
583    operand: &crate::RawStridedRef<'_, T>,
584    start_indices: &crate::RawStridedRef<'_, I>,
585    exec: Exec<'_>,
586) -> Result<()>
587where
588    T: Copy + Send + Sync,
589    I: crate::GatherIndex,
590{
591    plan.execute_uninit(dest, operand, start_indices, exec)
592}
593
594/// Full-overwrite indexed replay for an operation-family adapter.
595///
596/// # Safety
597/// The adapter must reject input/output overlap before creating the input
598/// references. All descriptor allocation and initialization contracts must
599/// remain valid for the call. Output slots may be uninitialized; the owning
600/// plan retains its checked layout/index validation and private initialization
601/// receipt. No initialized reference to the output backing may be formed.
602///
603/// # Examples
604///
605/// ```
606/// use strided_basic::execution::*;
607/// // SAFETY: the example supplies matching bounded layouts and disjoint storage.
608/// use strided_basic::{DynamicSlicePlan, RawStridedRef, RawStridedMut};
609/// use core::mem::MaybeUninit;
610/// let plan = DynamicSlicePlan::compile(&[3], &[1], &[1], &[1], &[2], &[1], &[2]).unwrap();
611/// let src = RawStridedRef::new(&[1_i32, 2, 3], &[3], &[1], 0).unwrap();
612/// let starts = RawStridedRef::new(&[1_i32], &[1], &[1], 0).unwrap();
613/// let mut values = [MaybeUninit::<i32>::uninit(); 2];
614/// let mut dst = RawStridedMut::new(&mut values, &[2], &[1], 0).unwrap();
615/// unsafe { dynamic_slice_into_uninit(&plan, &mut dst, &src, &starts, strided_basic::Exec::serial()) }.unwrap();
616/// assert_eq!(unsafe { values[1].assume_init() }, 3);
617/// ```
618///
619/// # Errors
620///
621/// Returns `StridedError::PlanLayoutMismatch` when the destination, operand,
622/// or start view does not carry the plan's compiled extents and strides,
623/// before any destination write. That is the only rejection: an empty
624/// operand returns before any offset arithmetic, and a non-empty one carries
625/// the bounds its constructor validated, so the replay arithmetic cannot
626/// overflow. Out-of-range start values are clamped, not rejected.
627#[inline]
628pub unsafe fn dynamic_slice_into_uninit<T, I>(
629    plan: &crate::DynamicSlicePlan,
630    dest: &mut crate::RawStridedMut<'_, core::mem::MaybeUninit<T>>,
631    operand: &crate::RawStridedRef<'_, T>,
632    starts: &crate::RawStridedRef<'_, I>,
633    exec: Exec<'_>,
634) -> Result<()>
635where
636    T: Copy + Send + Sync,
637    I: crate::GatherIndex,
638{
639    plan.execute_uninit(dest, operand, starts, exec)
640}
641
642/// Full-overwrite indexed replay for an operation-family adapter.
643///
644/// # Safety
645/// The adapter must reject input/output overlap before creating the input
646/// references. All descriptor allocation and initialization contracts must
647/// remain valid for the call. Output slots may be uninitialized; the owning
648/// plan retains its checked layout/index validation and private initialization
649/// receipt. No initialized reference to the output backing may be formed.
650/// Copy the operand and overwrite its selected window.
651///
652/// # Examples
653///
654/// ```
655/// use strided_basic::{RawStridedRef, RawStridedMut};
656/// use strided_basic::execution::*;
657/// use core::mem::MaybeUninit;
658/// use strided_basic::DynamicUpdateSlicePlan;
659/// let plan = DynamicUpdateSlicePlan::compile(&[3], &[1], &[1], &[1], &[1], &[1], &[3], &[1]).unwrap();
660/// let src = RawStridedRef::new(&[1_i32, 2, 3], &[3], &[1], 0).unwrap();
661/// let update = RawStridedRef::new(&[9_i32], &[1], &[1], 0).unwrap();
662/// let starts = RawStridedRef::new(&[1_i32], &[1], &[1], 0).unwrap();
663/// let mut values = [MaybeUninit::<i32>::uninit(); 3];
664/// let mut dst = RawStridedMut::new(&mut values, &[3], &[1], 0).unwrap();
665/// // SAFETY: all inputs are initialized and disjoint from the matching output.
666/// unsafe { dynamic_update_into_uninit(&plan, &mut dst, &src, &update, &starts, strided_basic::Exec::serial()) }.unwrap();
667/// assert_eq!(unsafe { values[1].assume_init() }, 9);
668/// ```
669///
670/// # Errors
671///
672/// Returns `StridedError::PlanLayoutMismatch` when the destination, operand,
673/// update, or start view does not carry the plan's compiled extents and
674/// strides, before any destination write. The clamped update window is
675/// preflighted next, still before any destination write, and rejects a window
676/// that does not fit `isize`: an empty destination returns before that offset
677/// arithmetic, and a non-empty one carries the bounds its constructor
678/// validated, so a checked view never reaches that rejection. Out-of-range
679/// start values are clamped, not rejected.
680#[inline]
681pub unsafe fn dynamic_update_into_uninit<'a, T, I>(
682    plan: &crate::DynamicUpdateSlicePlan,
683    dest: &'a mut crate::RawStridedMut<'a, core::mem::MaybeUninit<T>>,
684    operand: &crate::RawStridedRef<'_, T>,
685    update: &crate::RawStridedRef<'_, T>,
686    starts: &crate::RawStridedRef<'_, I>,
687    exec: Exec<'_>,
688) -> Result<()>
689where
690    T: Copy + Send + Sync,
691    I: crate::GatherIndex,
692{
693    plan.execute_uninit(dest, operand, update, starts, exec)
694}
695
696/// Full-overwrite indexed replay for an operation-family adapter.
697///
698/// # Safety
699/// The adapter must reject input/output overlap before creating the input
700/// references. All descriptor allocation and initialization contracts must
701/// remain valid for the call. Output slots may be uninitialized; the owning
702/// plan retains its checked layout/index validation and private initialization
703/// receipt. No initialized reference to the output backing may be formed.
704/// Copy the operand before additive scatter into its initialized logical slots.
705///
706/// # Examples
707///
708/// ```
709/// use strided_basic::{RawStridedRef, RawStridedMut};
710/// use strided_basic::execution::*;
711/// use core::mem::MaybeUninit;
712/// use strided_basic::{ScatterPlan, ScatterSpec};
713/// let spec = ScatterSpec { update_window_dims: vec![], inserted_window_dims: vec![0], scatter_dims_to_operand_dims: vec![0], index_vector_dim: 1 };
714/// let plan = ScatterPlan::compile(&[3], &[1], &[1, 1], &[1, 1], &[1], &[1], &[3], &[1], spec).unwrap();
715/// let src = RawStridedRef::new(&[1_i32, 2, 3], &[3], &[1], 0).unwrap();
716/// let update = RawStridedRef::new(&[9_i32], &[1], &[1], 0).unwrap();
717/// let indices = RawStridedRef::new(&[1_i64], &[1, 1], &[1, 1], 0).unwrap();
718/// let mut values = [MaybeUninit::<i32>::uninit(); 3];
719/// let mut dst = RawStridedMut::new(&mut values, &[3], &[1], 0).unwrap();
720/// // SAFETY: all inputs are initialized and disjoint from the matching output.
721/// unsafe { scatter_into_uninit(&plan, &mut dst, &src, &indices, &update, i32::wrapping_add, strided_basic::Exec::serial()) }.unwrap();
722/// assert_eq!(unsafe { values[1].assume_init() }, 11);
723/// ```
724///
725/// # Errors
726///
727/// Returns `StridedError::PlanLayoutMismatch` when the destination, operand,
728/// scatter-index, or update view does not carry the plan's compiled extents
729/// and strides, before any destination write. That is the only rejection: an
730/// empty batch or window skips the replay entirely, so no offset arithmetic
731/// runs, and a non-empty one carries the bounds its constructor validated.
732/// Out-of-range scatter indices are clamped, not rejected.
733#[inline]
734pub unsafe fn scatter_into_uninit<'a, T, I>(
735    plan: &crate::ScatterPlan,
736    dest: &'a mut crate::RawStridedMut<'a, core::mem::MaybeUninit<T>>,
737    operand: &crate::RawStridedRef<'_, T>,
738    scatter_indices: &crate::RawStridedRef<'_, I>,
739    updates: &crate::RawStridedRef<'_, T>,
740    combine: fn(T, T) -> T,
741    exec: Exec<'_>,
742) -> Result<()>
743where
744    T: Copy + core::ops::Add<Output = T> + Send + Sync,
745    I: crate::GatherIndex,
746{
747    plan.execute_uninit(dest, operand, scatter_indices, updates, combine, exec)
748}