Skip to main content

rustymath_core/
convert.rs

1//! Conversions between floats and integers whose loops vectorize on every target. AVX2 and wasm have no vector instruction for converting between `f64` and a 64-bit integer. AArch64 converts bytes to floats two lanes at a time. A loop that converts such types with `as` runs one item at a time on those targets. [`int`] and [`float`] convert them through a float's bits. They use the conversion instruction for types that every target converts in vector form, and for 64-bit integers where the target can. [`count`] writes runs of whole numbers as floats. Where the target can't convert 64-bit integers in vector form, it converts one item in 8. [`saturated`] also converts big integers and `u64`s, saturating them at an integer type's range.
2use crate::float::{Float, sealed::{Core, Sealed}};
3use num_bigint::{BigInt, Sign};
4use num_traits::Bounded;
5
6/// An integer type that [`int`] and [`float`] convert: `i8`, `i16`, `i32`, `i64`, `u8`, `u16` or `u32`.
7pub trait Int: Copy + Sealed + sealed::Ways {}
8
9#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
10pub enum ConversionKind { Cast, Truncate, Bits, Keep, Floor }
11
12/// Resolved kinds include the working float width even when the other kind is a word.
13#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
14pub struct Conversion { pub input: crate::buffer::DType, pub output: crate::buffer::DType, pub kind: ConversionKind }
15
16macro_rules! conversion {
17    ($call:expr, $checked:expr, $tolerance:expr, $write:ident) => {{
18        use crate::{buffer::{DType, Element, FloatWidth, Width}, error::{Error, Kind}};
19        match ($call.kind, $call.input, $call.output) {
20            (ConversionKind::Floor, DType::Float(width), DType::Integer(to)) => with_width!(to, I => with_float_width!(width, F => {
21                crate::with_tolerance!($tolerance, |t| $write!(F, I, |x| Ok(floored(x, t))))
22            })),
23            (ConversionKind::Floor, DType::Integer(from), DType::Integer(to)) => {
24                with_width!(to, I => with_width!(from, S => $write!(S, I, |x| Ok(x as I))))
25            }
26            (ConversionKind::Floor | ConversionKind::Truncate, DType::Boolean, DType::Integer(to)) => {
27                with_width!(to, I => $write!(bool, I, |x| Ok(u8::from(x) as I)))
28            }
29            (ConversionKind::Truncate, DType::Float(width), DType::Integer(to)) => {
30                with_width!(to, I => with_float_width!(width, F => $write!(F, I, |x| Ok(saturated(x)))))
31            }
32            (ConversionKind::Cast | ConversionKind::Truncate, DType::Integer(from), DType::Integer(to)) => with_width!(to, I => with_width!(from, S => {
33                $write!(S, I, |x| {
34                    if $checked { I::try_from(x).map_err(|_| Error::new(Kind::Overflow, "integer doesn't fit its converted dtype")) } else { Ok(x as I) }
35                })
36            })),
37            (ConversionKind::Truncate, input_type, DType::Character) if input_type.is_integral() => $write!(i64, char, |x| {
38                match u32::try_from(x).ok().and_then(char::from_u32) {
39                    Some(x) => Ok(x), None if $checked => Err(Error::new(Kind::Domain, "result is not a Unicode character")), None => Ok(char::FILL)
40                }
41            }),
42            (ConversionKind::Bits, DType::Float(FloatWidth::F32), DType::Integer(Width::I64)) => {
43                $write!(f32, i64, |x| Ok(i64::from(x.to_bits())))
44            }
45            (ConversionKind::Bits, DType::Float(FloatWidth::F64), DType::Integer(Width::I64)) => {
46                $write!(f64, i64, |x| Ok(x.to_bits() as i64))
47            }
48            (ConversionKind::Bits, DType::Integer(_), DType::Float(FloatWidth::F32)) => {
49                $write!(i64, f32, |x| Ok(f32::from_bits(x as u32)))
50            }
51            (ConversionKind::Bits, DType::Integer(_), DType::Float(FloatWidth::F64)) => {
52                $write!(i64, f64, |x| Ok(f64::from_bits(x as u64)))
53            }
54            (ConversionKind::Keep, _, dtype) => with_dtype!(dtype, T => $write!(@keep T)),
55            (ConversionKind::Cast | ConversionKind::Floor | ConversionKind::Truncate, _, dtype) => with_dtype!(dtype, T => $write!(@cast T)),
56            _ => $write!(@unsupported),
57        }
58    }};
59}
60impl Conversion {
61    pub fn validity<F: crate::Float, O: crate::recipe::Ops<F> + ?Sized>(self, o: &O, x: O::V) -> Option<(crate::error::Kind, O::B)> {
62        use crate::buffer::{DType, Width};
63        if self.kind == ConversionKind::Keep && self.input.is_integral() && self.output.is_integral() {
64            let bounds = |dtype| match dtype {
65                DType::Boolean => (0, 1), DType::Integer(width) => with_width!(width, T => (T::MIN as i64, T::MAX as i64)), _ => unreachable!()
66            };
67            let ((input_lo, input_hi), (lo, hi)) = (bounds(self.input), bounds(self.output));
68            if lo <= input_lo && hi >= input_hi { return None; }
69            let x = o.to_integer(x);
70            let valid = o.and(
71                o.integer_compare(crate::recipe::Cmp::LessEqual, o.integer(lo.max(input_lo)), x),
72                o.integer_compare(crate::recipe::Cmp::LessEqual, x, o.integer(hi.min(input_hi))));
73            return Some((crate::error::Kind::Domain, valid));
74        }
75        if self.kind != ConversionKind::Cast { return None; }
76        let valid = match self.output {
77            DType::Boolean => o.or(o.equal(x, o.number(F::zero())), o.equal(x, o.number(F::one()))),
78            DType::Integer(width) if matches!(self.input, DType::Float(_)) => {
79                let upper = if width == Width::U8 { 256.0 } else { 2f64.powi((8 * self.output.bytes() - 1) as i32) };
80                let lower = if width == Width::U8 { 0.0 } else { -upper };
81                let x = if width == Width::I64 { x } else { o.from_integer(o.to_integer(x)) };
82                o.and(o.less_equal(o.number(F::narrow(lower)), x), o.less(x, o.number(F::narrow(upper))))
83            }
84            _ => return None,
85        };
86        Some((crate::error::Kind::Overflow, valid))
87    }
88    pub fn may_raise(self, checked: bool) -> bool {
89        self.kind == ConversionKind::Keep || checked && (self.output == crate::buffer::DType::Character || self
90                        .validity::<f64, _>(&crate::recipe::RangeOps(self.input), crate::domain::Range::of_dtype(self.input))
91                        .is_some_and(|(_, valid)| valid != Some(true)))
92    }
93    /// Converts contiguous storage through the shared typed conversion rules, without a read plan.
94    pub(crate) fn buffer(self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance) -> Result<crate::buffer::Buffer, crate::error::Error> {
95        use crate::buffer::Cast;
96        macro_rules! write {
97            ($s:ty, $t:ty, $convert:expr) => { Ok(converted_buffer::<$s, $t>(view, checked, $convert)) };
98            (@keep $t:ty) => { Ok(<$t>::buffer((0..view.len()).map(|i| converted_item(crate::buffer::kept_as::<$t>(view, i)).0).collect())) };
99            (@cast $t:ty) => {{
100                let mut values = Vec::with_capacity(view.len());
101                if let Err(error) = <$t>::extend_cast(view, view.len(), checked, &mut values) { crate::error::record(error); values.resize(view.len(), <$t>::FILL); }
102                Ok(<$t>::buffer(values))
103            }};
104            (@unsupported) => { Err(crate::error::Error::new(crate::error::Kind::Domain, "unsupported resolved conversion")) };
105        }
106        crate::error::checked(|| conversion!(self, checked, tolerance, write))
107    }
108    pub(crate) fn append<'a, 'b>(
109        self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance,
110        scratch: &'a mut crate::buffer::Scratch, output: crate::buffer::MapOutput<'a, 'b>
111    ) -> Result<(), crate::error::Error> {
112        let sources = [crate::agreement::Binding::full(view)];
113        let plan = crate::agreement::ReadPlan::input(0, view.len());
114        self.map_in(crate::agreement::Read { sources: &sources, plan: &plan }, 0..view.len(), checked, tolerance, scratch, output)
115    }
116    pub(crate) fn map(
117        self, input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, output: &mut crate::buffer::Buffer
118    ) -> Result<(), crate::error::Error> {
119        if output.dtype() != self.output { *output = crate::buffer::Buffer::with_capacity(self.output, range.len()); } else { output.clear(); }
120        self.map_in(input, range, checked, crate::tolerant::Tolerance::EXACT, &mut crate::buffer::Scratch::default(), crate::buffer::MapOutput::Buffer(output))
121    }
122
123    pub fn map_in<'a, 'b>(
124        self, input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, tolerance: crate::tolerant::Tolerance,
125        scratch: &'a mut crate::buffer::Scratch, target: crate::buffer::MapOutput<'a, 'b>
126    ) -> Result<(), crate::error::Error> {
127        let mut output = crate::buffer::Output { target, result_dtype: self.output, checked, scratch: Some(scratch) };
128        macro_rules! write {
129            ($s:ty, $t:ty, $convert:expr) => { written::<$s, $t>(input, range, checked, &mut output, $convert) };
130            (@cast $t:ty) => { write!($t, $t, Ok) };
131            (@keep $t:ty) => { kept_mapped::<$t>(self.input, input, range, checked, &mut output) };
132            (@unsupported) => {{ output.target.fill(range.len()); Err(Error::new(Kind::Domain, "unsupported resolved conversion")) }};
133        }
134        crate::error::checked(|| conversion!(self, checked, tolerance, write))
135    }
136}
137
138#[inline]
139fn converted_item<T: crate::buffer::Element>(value: Result<T, crate::error::Error>) -> (T, bool) {
140    match value { Ok(value) => (value, true), Err(error) => { crate::error::record(error); (T::FILL, false) } }
141}
142
143fn converted_buffer<S: crate::buffer::Cast, T: crate::buffer::Element>(
144    view: crate::buffer::View<'_>, checked: bool, convert: impl Fn(S) -> Result<T, crate::error::Error>
145) -> crate::buffer::Buffer {
146    let (mut values, mut scratch) = (Vec::with_capacity(view.len()), Vec::new());
147    for from in (0..view.len()).step_by(crate::CHUNK) {
148        let part = view.slice(from..(from + crate::CHUNK).min(view.len()));
149        let items = if let Some(items) = S::of(part) { items } else {
150            scratch.clear();
151            if let Err(error) = S::extend_cast(part, part.len(), checked, &mut scratch) {
152                crate::error::record(error); values.resize(values.len() + part.len(), T::FILL); continue;
153            }
154            &scratch
155        };
156        values.extend(items.iter().map(|&x| converted_item(convert(x)).0));
157    }
158    T::buffer(values)
159}
160
161#[inline]
162fn written<S: crate::buffer::Cast, T: crate::buffer::OutputItem>(
163    input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, output: &mut crate::buffer::Output<'_, '_>,
164    convert: impl Fn(S) -> Result<T, crate::error::Error> + Sync
165) -> Result<(), crate::error::Error> {
166    output.write::<T>(range.len(), &|part, slots, work| {
167        let part = part.start + range.start..part.end + range.start;
168        let in_place = input.in_place::<S>(&part);
169        let len = if in_place { part.len().max(1) } else { crate::CHUNK };
170        let mut local = Vec::new();
171        let (mut work, mut copied) = (work, 0);
172        let scratch = match work.as_deref_mut() { Some(work) if !in_place => { let [scratch] = work.inputs::<S, 1>(); scratch }, _ => &mut local };
173        let ok = part.clone().step_by(len).fold(true, |ok, start| {
174            let range = start..(start + len).min(part.end);
175            let n = range.len();
176            let input = match input.part_counted::<S>(range, checked, scratch) {
177                Ok((input, n)) => { copied += n; input }
178                Err(error) => {
179                    crate::error::record(error);
180                    return unsafe { slots.fill(n, |out| { for slot in out { slot.write(T::FILL); } false }) } & ok;
181                }
182            };
183            slots.pairs(input, crate::Arg::One(()), n, |x, ()| converted_item(convert(x))) & ok
184        });
185        if let Some(work) = work { work.record_copies(copied); }
186        ok
187    })
188}
189
190pub(crate) mod sealed {
191    /// One integer type's conversions from and to `f32` and `f64`, which [`int`](super::int), [`saturated`](super::saturated) and [`float`](super::float) pick by the float type.
192    pub trait Ways: Copy {
193        fn from_f32(x: f32) -> Option<Self>;
194        fn from_f64(x: f64) -> Option<Self>;
195        fn saturated_f32(x: f32) -> Self;
196        fn saturated_f64(x: f64) -> Self;
197        fn to_f32(self) -> f32;
198        fn to_f64(self) -> f64;
199    }
200}
201
202/// `x`, a whole float, as an integer of type `I`. `None` where `x` is NaN or infinite, or lies outside `I`. For `i64` and `u32`, a magnitude beyond 2^51 - 1 also gives `None`, and so does `i32::MIN` from an `f32`. A fraction gives an integer next to it, or `None`.
203#[inline(always)]
204pub fn int<I: Int, F: Float>(x: F) -> Option<I> { if <F::Working as Core>::WIDE { I::from_f64(x.wide()) } else { I::from_f32(x.wide() as f32) } }
205
206/// `x` as an integer of type `I`, or `I`'s largest or smallest value where `x` lies past its range. A float converts as Rust's `as` converts it: NaN gives 0, and a fraction gives the integer toward 0. A type that fits an `i32` converts from a float in vector form on every target, where `as` would convert one item at a time, as it does from an `f64` on AArch64 and from any float on x86-64. [`Saturates`] lists the other sources.
207#[inline(always)]
208pub fn saturated<I, S: Saturates<I>>(x: S) -> I { x.saturated() }
209/// A number that [`saturated`] converts to the integer type `I`: a float to any [`Int`], and a big integer or a `u64` to any integer type that `TryFrom` converts it to.
210pub trait Saturates<I> { fn saturated(self) -> I; }
211impl<I: Int, F: Float> Saturates<I> for F {
212    #[inline(always)]
213    fn saturated(self) -> I { if <F::Working as Core>::WIDE { I::saturated_f64(self.wide()) } else { I::saturated_f32(self.wide() as f32) } }
214}
215impl<I: Bounded + for<'a> TryFrom<&'a BigInt>> Saturates<I> for &BigInt {
216    fn saturated(self) -> I { I::try_from(self).unwrap_or_else(|_| if self.sign() == Sign::Minus { I::min_value() } else { I::max_value() }) }
217}
218impl<I: Bounded + TryFrom<u64>> Saturates<I> for u64 { fn saturated(self) -> I { I::try_from(self).unwrap_or_else(|_| I::max_value()) } }
219/// The floor of `x` within `t` as an integer of type `I`, saturated at `I`'s range as [`saturated`] saturates: NaN gives 0. The one float-to-integer rule for conversions that change a number's domain: BPL's `ₓ` marks and Python's `astype` and integer `dtype=`.
220#[inline(always)]
221pub fn floored<I: Int, F: Float>(x: F, t: impl crate::tolerant::Tolerant) -> I { saturated(t.floor(x)) }
222
223/// `i` as the nearest float of type `F`.
224#[inline(always)]
225pub fn float<F: Float, I: Int>(i: I) -> F { if <F::Working as Core>::WIDE { F::narrow(i.to_f64()) } else { F::narrow(i.to_f32().into()) } }
226
227/// Appends the `n` whole numbers `start`, `start + 1`, `start + 2` and so on to `out` as floats of type `F`, or `start`, `start - 1` and so on where `down` is true. Each is the whole number rounded once to `F`. A float narrower than `f64` counts in `i32` where every item fits one, because every target converts `i32` to it in vector form. Otherwise a target with `NATIVE_64` converts each item from an integer, as [`float`] does. Wasm and x86-64 without AVX-512 have no vector instruction for that, so there each block of 8 items adds 0 to 7 steps to the block's first item, which comes from the block's position.
228pub fn count<F: Float>(out: &mut Vec<F>, start: i64, down: bool, n: usize) {
229    let end = if down { start - n as i64 } else { start + n as i64 };
230    if !<F::Working as Core>::WIDE && i32::try_from(start).is_ok() && i32::try_from(end).is_ok() {
231        let start = start as i32;
232        return if down { out.extend((0..n as i32).map(move |i| float::<F, i32>(start - i))) }
233        else { out.extend((0..n as i32).map(move |i| float::<F, i32>(start + i))) };
234    }
235    if NATIVE_64 {
236        return if down { out.extend((0..n).map(move |i| F::narrow(wide_float(start - i as i64)))) }
237        else { out.extend((0..n).map(move |i| F::narrow(wide_float(start + i as i64)))) };
238    }
239    let step = if down { -1 } else { 1 };
240    let offsets: [f64; 8] = std::array::from_fn(|k| wide_float(step * k as i64));
241    let first = move |j: usize| wide_float(start + step * (8 * j) as i64);
242    unsafe {
243        crate::slice::appended(out, n, |slots| {
244            let (blocks, rest) = slots.as_chunks_mut::<8>();
245            let whole = blocks.len();
246            for (j, block) in blocks.iter_mut().enumerate() { let first = first(j); for k in 0..8 { block[k].write(F::narrow(first + offsets[k])); } }
247            for (slot, &offset) in rest.iter_mut().zip(&offsets) { slot.write(F::narrow(first(whole) + offset)); }
248        })
249    };
250}
251
252/// Whether the target converts between `f64` and 64-bit integers in vector form, as AArch64 and x86-64 with AVX-512 do. Elsewhere they convert through a float's bits. Both ways give the same results for whole floats.
253const NATIVE_64: bool = cfg!(any(target_arch = "aarch64", target_feature = "avx512dq"));
254
255/// Whether the target converts floats to `i32` with saturation in vector form, as AArch64's `fcvtzs` and wasm's `trunc_sat` do. On x86-64, where Rust's saturating `as` compiles to one scalar conversion per item, floats convert without saturation after a range check.
256const SATURATING: bool = !cfg!(target_arch = "x86_64");
257
258/// The magnitude up to which the bits of `f64::SHIFT` + x hold the whole number x.
259pub(crate) const WIDE_LIMIT: f64 = 2251799813685247.0;
260
261/// `i` as the nearest `f64`, natively or through `halves`.
262#[inline(always)]
263fn wide_float(i: i64) -> f64 { if NATIVE_64 { i as f64 } else { halves(i) } }
264
265/// `i` as the nearest `f64` through bits. The high half converts through the bits of 2^84 + 2^63 plus it times 2^32, and the low half through `from_low`. Both convert exactly, and their sum rounds once.
266#[inline(always)]
267fn halves(i: i64) -> f64 {
268    let high = f64::from_bits(0x4530_0000_0000_0000 | ((i as u64 >> 32) ^ 0x8000_0000)) - f64::from_bits(0x4530_0000_8000_0000);
269    high + f64::from_low(i as u64 & 0xffff_ffff, 0.0)
270}
271
272/// The whole number nearest `x`, for a magnitude up to `WIDE_LIMIT`, through the bits of `f64::SHIFT` + x.
273#[inline(always)]
274pub(crate) fn shifted(x: f64) -> i64 { (x + f64::SHIFT).to_bits().wrapping_sub(f64::SHIFT.to_bits()) as i64 }
275
276/// The whole number `i`, of magnitude up to `WIDE_LIMIT`, as an `f64`, through the bits of `f64::SHIFT` + i. It undoes [`shifted`].
277#[inline(always)]
278pub(crate) fn unshifted(i: i64) -> f64 { f64::from_bits(f64::SHIFT.to_bits().wrapping_add(i as u64)) - f64::SHIFT }
279
280/// Writes each integer type's conversions. `narrow` types fit an `i32`, and every target converts a float to an `i32` in vector form. `wide` types convert from a float through the bits of `f64::SHIFT` + x, unless the target has `NATIVE_64`. The two expressions convert `i` to `f32` and `f64`.
281macro_rules! ints {
282    ($($t:ty: $kind:ident, |$i:ident| $f32:expr, $f64:expr;)+) => {$(
283        impl Int for $t {}
284        impl sealed::Ways for $t {
285            #[inline(always)]
286            fn from_f32(x: f32) -> Option<$t> { ints!(@$kind f32, $t, x) }
287            #[inline(always)]
288            fn from_f64(x: f64) -> Option<$t> { ints!(@$kind f64, $t, x) }
289            #[inline(always)]
290            fn saturated_f32(x: f32) -> $t { ints!(@saturated $kind f32, $t, x) }
291            #[inline(always)]
292            fn saturated_f64(x: f64) -> $t { ints!(@saturated $kind f64, $t, x) }
293            #[inline(always)]
294            fn to_f32(self) -> f32 { let $i = self; $f32 }
295            #[inline(always)]
296            fn to_f64(self) -> f64 { let $i = self; $f64 }
297        }
298    )+};
299    (@narrow $f:ty, $t:ty, $x:ident) => {{
300        // One comparison, of the distance from the middle of `$t`'s range, keeps the loop to one mask.
301        let (mid, half) = ((<$t>::MIN as f64 + <$t>::MAX as f64) / 2.0, (<$t>::MAX as f64 - <$t>::MIN as f64 + 1.0) / 2.0);
302        (($x - mid as $f).abs() < half as $f).then(|| {
303            if SATURATING { return $x as i32 as $t }
304            // SAFETY: the comparison excludes NaN, the infinities and every float that truncates outside `$t`, which `i32` holds. The conversion then needs no saturation.
305            unsafe { $x.to_int_unchecked::<i32>() as $t }
306        })
307    }};
308    (@wide f32, $t:ty, $x:ident) => { Self::from_f64($x.into()) };
309    (@wide f64, $t:ty, $x:ident) => {{
310        let n = if NATIVE_64 { $x as i64 } else { shifted($x) };
311        ($x.abs() <= WIDE_LIMIT).then_some(n).and_then(|n| <$t>::try_from(n).ok())
312    }};
313    (@saturated narrow f32, $t:ty, $x:ident) => { if SATURATING { $x as $t } else { ints!(@clamped f32, $t, $x) } };
314    // A target with `NATIVE_64` converts an `f64` to an `i64` with saturation in vector form, which the clamp then narrows as `as` would.
315    (@saturated narrow f64, $t:ty, $x:ident) => { if NATIVE_64 { ($x as i64).clamp(<$t>::MIN.into(), <$t>::MAX.into()) as $t } else { ints!(@clamped f64, $t, $x) } };
316    (@saturated wide $f:ty, $t:ty, $x:ident) => { $x as $t };
317    (@clamped $f:ty, $t:ty, $x:ident) => {{
318        // Where `$f` can't hold `$t::MAX`, as an `f32` can't hold `i32::MAX`, the maximum rounds up to a float outside `$t`. The clamp then stops one float below it, and a float at or past it gives `$t::MAX`.
319        let top = <$t>::MAX as $f;
320        let below = if top as i128 > <$t>::MAX as i128 { top.next_down() } else { top };
321        let c = if $x.is_nan() { 0.0 } else { $x.max(<$t>::MIN as $f).min(below) };
322        // SAFETY: `c` is never NaN, and the clamp keeps it within `$t`.
323        let i = unsafe { c.to_int_unchecked::<$t>() };
324        if $x >= top { <$t>::MAX } else { i }
325    }};
326}
327
328// A byte or a `u16` reads as a float through `from_low`. AArch64's own conversion handles bytes two lanes at a time.
329ints! {
330    i8: narrow, |i| i as f32, i as f64;
331    i16: narrow, |i| i as f32, i as f64;
332    i32: narrow, |i| i as f32, i as f64;
333    u8: narrow, |i| f32::from_low(i.into(), 0.0), f32::from_low(i.into(), 0.0).into();
334    u16: narrow, |i| f32::from_low(i.into(), 0.0), f32::from_low(i.into(), 0.0).into();
335    u32: wide, |i| f64::from_low(i.into(), 0.0) as f32, f64::from_low(i.into(), 0.0);
336    i64: wide, |i| wide_float(i) as f32, wide_float(i);
337}
338
339fn kept_mapped<T: crate::buffer::OutputItem + crate::buffer::Cast>(
340    dtype: crate::buffer::DType, input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, output: &mut crate::buffer::Output<'_, '_>
341) -> Result<(), crate::error::Error> {
342    output.write::<T>(range.len(), &|part, slots, work| {
343        let mut local = crate::buffer::Buffer::with_capacity(dtype, 0);
344        let scratch = match work { Some(work) => work.buffer(dtype), None => &mut local };
345        match input.part_view(part.start + range.start..part.end + range.start, dtype, checked, scratch) {
346            Ok(values) => unsafe { slots.fill(part.len(), |out| {
347                let mut good = true;
348                for (i, slot) in out.iter_mut().enumerate() {
349                    slot.write(match crate::buffer::kept_as::<T>(values, i) {
350                        Ok(value) => value, Err(error) => { crate::error::record(error); good = false; T::FILL }
351                    });
352                }
353                good
354            }) },
355            Err(error) => {
356                crate::error::record(error);
357                unsafe { slots.fill(part.len(), |out| { for slot in out { slot.write(T::FILL); } false }) }
358            }
359        }
360    })
361}