1use crate::kernel::{
22 ensure_same_shape, for_each_inner_block_preordered, pointwise_plan,
23 sequential_contiguous_layout, total_len,
24};
25use crate::layout_check::is_injective_layout;
26use crate::simd;
27use crate::view::{StridedView, StridedViewMut};
28use crate::{Exec, Result, StridedError};
29use strided_view::ElementOp;
30
31use crate::cost::KernelClass;
32use crate::fuse::compute_costs;
33use crate::threading::{for_each_inner_block_with_offsets, is_parallel, mapreduce_threaded};
34
35type Body<'a> = &'a (dyn Fn(&[isize], usize, &[isize]) + Sync);
37
38struct Raw<T>(*mut T);
41impl<T> Clone for Raw<T> {
42 fn clone(&self) -> Self {
43 *self
44 }
45}
46impl<T> Copy for Raw<T> {}
47unsafe impl<T> Send for Raw<T> {}
49unsafe impl<T> Sync for Raw<T> {}
50impl<T> Raw<T> {
51 fn get(self) -> *mut T {
53 self.0
54 }
55}
56
57fn validate_destination(dims: &[usize], strides: &[isize]) -> Result<()> {
58 if is_injective_layout(dims, strides) {
59 Ok(())
60 } else {
61 Err(StridedError::NonInjectiveOutputLayout)
62 }
63}
64
65fn run_update(
77 dims: &[usize],
78 strides_list: &[&[isize]],
79 elem_sizes: &[usize],
80 body: Body<'_>,
81 exec: Exec<'_>,
82) -> Result<()> {
83 let total = total_len(dims)?;
84 if total == 0 {
85 return Ok(());
86 }
87 if sequential_contiguous_layout(KernelClass::Map, dims, strides_list, &exec)?.is_some() {
92 let zeros: crate::inline::OperandVec<isize> = strides_list.iter().map(|_| 0isize).collect();
93 let ones: crate::inline::OperandVec<isize> = strides_list.iter().map(|_| 1isize).collect();
94 body(&zeros, total, &ones);
95 return Ok(());
96 }
97 if dims.iter().all(|&d| d == 1) {
99 let zeros: crate::inline::OperandVec<isize> = strides_list.iter().map(|_| 0isize).collect();
100 body(&zeros, 1, &zeros);
101 return Ok(());
102 }
103 let (fused_dims, ordered_strides, plan) =
104 pointwise_plan(dims, strides_list, Some(0), elem_sizes, total);
105
106 {
107 let total = total_len(&fused_dims)?;
108 if is_parallel(KernelClass::Map, total, &exec) {
109 let costs = compute_costs(&ordered_strides);
110 let initial_offsets: crate::inline::OperandVec<isize> =
111 strides_list.iter().map(|_| 0isize).collect();
112 return mapreduce_threaded(
113 &fused_dims,
114 &plan.block,
115 &ordered_strides,
116 &initial_offsets,
117 &costs,
118 KernelClass::Map,
119 exec,
120 &|dims, blocks, strides_list, offsets| {
121 for_each_inner_block_with_offsets(
122 dims,
123 blocks,
124 strides_list,
125 offsets,
126 |offsets, len, strides| {
127 body(offsets, len, strides);
128 Ok(())
129 },
130 )
131 },
132 );
133 }
134 }
135
136 let initial_offsets: crate::inline::OperandVec<isize> =
137 ordered_strides.iter().map(|_| 0isize).collect();
138 for_each_inner_block_preordered(
139 &fused_dims,
140 &plan.block,
141 &ordered_strides,
142 &initial_offsets,
143 |offsets, len, strides| {
144 body(offsets, len, strides);
145 Ok(())
146 },
147 )
148}
149
150#[inline(always)]
157unsafe fn inner_loop_update1<D: Copy, OpD: ElementOp<D>>(
158 dp: *mut D,
159 ds: isize,
160 len: usize,
161 f: &impl Fn(D) -> D,
162) {
163 if ds == 1 {
164 let dst = std::slice::from_raw_parts_mut(dp, len);
165 simd::dispatch_if_large(len, || {
166 for d in dst.iter_mut() {
167 *d = f(OpD::apply(*d));
168 }
169 });
170 } else {
171 let mut dp = dp;
172 for _ in 0..len {
173 *dp = f(OpD::apply(*dp));
174 dp = dp.wrapping_offset(ds);
175 }
176 }
177}
178
179#[inline(always)]
185unsafe fn inner_loop_update2<D: Copy, A: Copy, OpD: ElementOp<D>, OpA: ElementOp<A>>(
186 dp: *mut D,
187 ds: isize,
188 ap: *const A,
189 a_s: isize,
190 len: usize,
191 f: &impl Fn(D, A) -> D,
192) {
193 if ds == 1 && a_s == 1 {
194 let src_a = std::slice::from_raw_parts(ap, len);
195 let dst = std::slice::from_raw_parts_mut(dp, len);
196 simd::dispatch_if_large(len, || {
197 for (d, &a) in dst.iter_mut().zip(src_a) {
198 *d = f(OpD::apply(*d), OpA::apply(a));
199 }
200 });
201 } else {
202 let (mut dp, mut ap) = (dp, ap);
203 for _ in 0..len {
204 *dp = f(OpD::apply(*dp), OpA::apply(*ap));
205 dp = dp.wrapping_offset(ds);
206 ap = ap.wrapping_offset(a_s);
207 }
208 }
209}
210
211#[inline(always)]
216#[allow(clippy::too_many_arguments)] unsafe fn inner_loop_update3<
218 D: Copy,
219 A: Copy,
220 B: Copy,
221 OpD: ElementOp<D>,
222 OpA: ElementOp<A>,
223 OpB: ElementOp<B>,
224>(
225 dp: *mut D,
226 ds: isize,
227 ap: *const A,
228 a_s: isize,
229 bp: *const B,
230 b_s: isize,
231 len: usize,
232 f: &impl Fn(D, A, B) -> D,
233) {
234 if ds == 1 && a_s == 1 && b_s == 1 {
235 let src_a = std::slice::from_raw_parts(ap, len);
236 let src_b = std::slice::from_raw_parts(bp, len);
237 let dst = std::slice::from_raw_parts_mut(dp, len);
238 simd::dispatch_if_large(len, || {
239 for ((d, &a), &b) in dst.iter_mut().zip(src_a).zip(src_b) {
240 *d = f(OpD::apply(*d), OpA::apply(a), OpB::apply(b));
241 }
242 });
243 } else {
244 let (mut dp, mut ap, mut bp) = (dp, ap, bp);
245 for _ in 0..len {
246 *dp = f(OpD::apply(*dp), OpA::apply(*ap), OpB::apply(*bp));
247 dp = dp.wrapping_offset(ds);
248 ap = ap.wrapping_offset(a_s);
249 bp = bp.wrapping_offset(b_s);
250 }
251 }
252}
253
254pub fn map_update_into<D, OpD>(
272 dest: &mut StridedViewMut<D>,
273 f: impl Fn(D) -> D + Sync,
274 exec: Exec<'_>,
275) -> Result<()>
276where
277 D: Copy + Send + Sync,
278 OpD: ElementOp<D>,
279{
280 validate_destination(dest.dims(), dest.strides())?;
281 let dp = Raw(dest.as_mut_ptr());
282 run_update(
283 dest.dims(),
284 &[dest.strides()],
285 &[std::mem::size_of::<D>()],
286 &|offsets, len, strides| {
287 unsafe {
289 inner_loop_update1::<D, OpD>(dp.get().offset(offsets[0]), strides[0], len, &f);
290 }
291 },
292 exec,
293 )
294}
295
296pub fn zip_update2_into<D, A, OpD, OpA>(
317 dest: &mut StridedViewMut<D>,
318 a: &StridedView<A, OpA>,
319 f: impl Fn(D, A) -> D + Sync,
320 exec: Exec<'_>,
321) -> Result<()>
322where
323 D: Copy + Send + Sync,
324 A: Copy + Send + Sync,
325 OpD: ElementOp<D>,
326 OpA: ElementOp<A>,
327{
328 ensure_same_shape(dest.dims(), a.dims())?;
329 validate_destination(dest.dims(), dest.strides())?;
330 let dp = Raw(dest.as_mut_ptr());
331 let ap = Raw(a.ptr() as *mut A);
332 run_update(
333 dest.dims(),
334 &[dest.strides(), a.strides()],
335 &[std::mem::size_of::<D>(), std::mem::size_of::<A>()],
336 &|offsets, len, strides| {
337 unsafe {
339 inner_loop_update2::<D, A, OpD, OpA>(
340 dp.get().offset(offsets[0]),
341 strides[0],
342 ap.get().offset(offsets[1]).cast_const(),
343 strides[1],
344 len,
345 &f,
346 );
347 }
348 },
349 exec,
350 )
351}
352
353pub fn zip_update3_into<D, A, B, OpD, OpA, OpB>(
377 dest: &mut StridedViewMut<D>,
378 a: &StridedView<A, OpA>,
379 b: &StridedView<B, OpB>,
380 f: impl Fn(D, A, B) -> D + Sync,
381 exec: Exec<'_>,
382) -> Result<()>
383where
384 D: Copy + Send + Sync,
385 A: Copy + Send + Sync,
386 B: Copy + Send + Sync,
387 OpD: ElementOp<D>,
388 OpA: ElementOp<A>,
389 OpB: ElementOp<B>,
390{
391 ensure_same_shape(dest.dims(), a.dims())?;
392 ensure_same_shape(dest.dims(), b.dims())?;
393 validate_destination(dest.dims(), dest.strides())?;
394 let dp = Raw(dest.as_mut_ptr());
395 let ap = Raw(a.ptr() as *mut A);
396 let bp = Raw(b.ptr() as *mut B);
397 run_update(
398 dest.dims(),
399 &[dest.strides(), a.strides(), b.strides()],
400 &[
401 std::mem::size_of::<D>(),
402 std::mem::size_of::<A>(),
403 std::mem::size_of::<B>(),
404 ],
405 &|offsets, len, strides| {
406 unsafe {
408 inner_loop_update3::<D, A, B, OpD, OpA, OpB>(
409 dp.get().offset(offsets[0]),
410 strides[0],
411 ap.get().offset(offsets[1]).cast_const(),
412 strides[1],
413 bp.get().offset(offsets[2]).cast_const(),
414 strides[2],
415 len,
416 &f,
417 );
418 }
419 },
420 exec,
421 )
422}
423
424#[cfg(test)]
425#[path = "update_view/tests/tests.rs"]
426mod tests;