1use crate::float::{Float, sealed::{Core, Sealed}};
3use num_bigint::{BigInt, Sign};
4use num_traits::Bounded;
5
6pub trait Int: Copy + Sealed + sealed::Ways {}
8
9#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
10pub enum ConversionKind { Cast, Truncate, Bits, Keep, Floor }
11
12#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
14pub struct Conversion { pub input: crate::buffer::DType, pub output: crate::buffer::DType, pub kind: ConversionKind }
15
16macro_rules! conversion {
17 ($call:expr, $checked:expr, $tolerance:expr, $write:ident) => {{
18 use crate::{buffer::{DType, Element, FloatWidth, Width}, error::{Error, Kind}};
19 match ($call.kind, $call.input, $call.output) {
20 (ConversionKind::Floor, DType::Float(width), DType::Integer(to)) => with_width!(to, I => with_float_width!(width, F => {
21 crate::with_tolerance!($tolerance, |t| $write!(F, I, |x| Ok(floored(x, t))))
22 })),
23 (ConversionKind::Floor, DType::Integer(from), DType::Integer(to)) => {
24 with_width!(to, I => with_width!(from, S => $write!(S, I, |x| Ok(x as I))))
25 }
26 (ConversionKind::Floor | ConversionKind::Truncate, DType::Boolean, DType::Integer(to)) => {
27 with_width!(to, I => $write!(bool, I, |x| Ok(u8::from(x) as I)))
28 }
29 (ConversionKind::Truncate, DType::Float(width), DType::Integer(to)) => {
30 with_width!(to, I => with_float_width!(width, F => $write!(F, I, |x| Ok(saturated(x)))))
31 }
32 (ConversionKind::Cast | ConversionKind::Truncate, DType::Integer(from), DType::Integer(to)) => with_width!(to, I => with_width!(from, S => {
33 $write!(S, I, |x| {
34 if $checked { I::try_from(x).map_err(|_| Error::new(Kind::Overflow, "integer doesn't fit its converted dtype")) } else { Ok(x as I) }
35 })
36 })),
37 (ConversionKind::Truncate, input_type, DType::Character) if input_type.is_integral() => $write!(i64, char, |x| {
38 match u32::try_from(x).ok().and_then(char::from_u32) {
39 Some(x) => Ok(x), None if $checked => Err(Error::new(Kind::Domain, "result is not a Unicode character")), None => Ok(char::FILL)
40 }
41 }),
42 (ConversionKind::Bits, DType::Float(FloatWidth::F32), DType::Integer(Width::I64)) => {
43 $write!(f32, i64, |x| Ok(i64::from(x.to_bits())))
44 }
45 (ConversionKind::Bits, DType::Float(FloatWidth::F64), DType::Integer(Width::I64)) => {
46 $write!(f64, i64, |x| Ok(x.to_bits() as i64))
47 }
48 (ConversionKind::Bits, DType::Integer(_), DType::Float(FloatWidth::F32)) => {
49 $write!(i64, f32, |x| Ok(f32::from_bits(x as u32)))
50 }
51 (ConversionKind::Bits, DType::Integer(_), DType::Float(FloatWidth::F64)) => {
52 $write!(i64, f64, |x| Ok(f64::from_bits(x as u64)))
53 }
54 (ConversionKind::Keep, _, dtype) => with_dtype!(dtype, T => $write!(@keep T)),
55 (ConversionKind::Cast | ConversionKind::Floor | ConversionKind::Truncate, _, dtype) => with_dtype!(dtype, T => $write!(@cast T)),
56 _ => $write!(@unsupported),
57 }
58 }};
59}
60impl Conversion {
61 pub fn validity<F: crate::Float, O: crate::recipe::Ops<F> + ?Sized>(self, o: &O, x: O::V) -> Option<(crate::error::Kind, O::B)> {
62 use crate::buffer::{DType, Width};
63 if self.kind == ConversionKind::Keep && self.input.is_integral() && self.output.is_integral() {
64 let bounds = |dtype| match dtype {
65 DType::Boolean => (0, 1), DType::Integer(width) => with_width!(width, T => (T::MIN as i64, T::MAX as i64)), _ => unreachable!()
66 };
67 let ((input_lo, input_hi), (lo, hi)) = (bounds(self.input), bounds(self.output));
68 if lo <= input_lo && hi >= input_hi { return None; }
69 let x = o.to_integer(x);
70 let valid = o.and(
71 o.integer_compare(crate::recipe::Cmp::LessEqual, o.integer(lo.max(input_lo)), x),
72 o.integer_compare(crate::recipe::Cmp::LessEqual, x, o.integer(hi.min(input_hi))));
73 return Some((crate::error::Kind::Domain, valid));
74 }
75 if self.kind != ConversionKind::Cast { return None; }
76 let valid = match self.output {
77 DType::Boolean => o.or(o.equal(x, o.number(F::zero())), o.equal(x, o.number(F::one()))),
78 DType::Integer(width) if matches!(self.input, DType::Float(_)) => {
79 let upper = if width == Width::U8 { 256.0 } else { 2f64.powi((8 * self.output.bytes() - 1) as i32) };
80 let lower = if width == Width::U8 { 0.0 } else { -upper };
81 let x = if width == Width::I64 { x } else { o.from_integer(o.to_integer(x)) };
82 o.and(o.less_equal(o.number(F::narrow(lower)), x), o.less(x, o.number(F::narrow(upper))))
83 }
84 _ => return None,
85 };
86 Some((crate::error::Kind::Overflow, valid))
87 }
88 pub fn may_raise(self, checked: bool) -> bool {
89 self.kind == ConversionKind::Keep || checked && (self.output == crate::buffer::DType::Character || self
90 .validity::<f64, _>(&crate::recipe::RangeOps(self.input), crate::domain::Range::of_dtype(self.input))
91 .is_some_and(|(_, valid)| valid != Some(true)))
92 }
93 pub(crate) fn buffer(self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance) -> Result<crate::buffer::Buffer, crate::error::Error> {
95 use crate::buffer::Cast;
96 macro_rules! write {
97 ($s:ty, $t:ty, $convert:expr) => { Ok(converted_buffer::<$s, $t>(view, checked, $convert)) };
98 (@keep $t:ty) => { Ok(<$t>::buffer((0..view.len()).map(|i| converted_item(crate::buffer::kept_as::<$t>(view, i)).0).collect())) };
99 (@cast $t:ty) => {{
100 let mut values = Vec::with_capacity(view.len());
101 if let Err(error) = <$t>::extend_cast(view, view.len(), checked, &mut values) { crate::error::record(error); values.resize(view.len(), <$t>::FILL); }
102 Ok(<$t>::buffer(values))
103 }};
104 (@unsupported) => { Err(crate::error::Error::new(crate::error::Kind::Domain, "unsupported resolved conversion")) };
105 }
106 crate::error::checked(|| conversion!(self, checked, tolerance, write))
107 }
108 pub(crate) fn append<'a, 'b>(
109 self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance,
110 scratch: &'a mut crate::buffer::Scratch, output: crate::buffer::MapOutput<'a, 'b>
111 ) -> Result<(), crate::error::Error> {
112 let sources = [crate::agreement::Binding::full(view)];
113 let plan = crate::agreement::ReadPlan::input(0, view.len());
114 self.map_in(crate::agreement::Read { sources: &sources, plan: &plan }, 0..view.len(), checked, tolerance, scratch, output)
115 }
116 pub(crate) fn map(
117 self, input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, output: &mut crate::buffer::Buffer
118 ) -> Result<(), crate::error::Error> {
119 if output.dtype() != self.output { *output = crate::buffer::Buffer::with_capacity(self.output, range.len()); } else { output.clear(); }
120 self.map_in(input, range, checked, crate::tolerant::Tolerance::EXACT, &mut crate::buffer::Scratch::default(), crate::buffer::MapOutput::Buffer(output))
121 }
122
123 pub fn map_in<'a, 'b>(
124 self, input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, tolerance: crate::tolerant::Tolerance,
125 scratch: &'a mut crate::buffer::Scratch, target: crate::buffer::MapOutput<'a, 'b>
126 ) -> Result<(), crate::error::Error> {
127 let mut output = crate::buffer::Output { target, result_dtype: self.output, checked, scratch: Some(scratch) };
128 macro_rules! write {
129 ($s:ty, $t:ty, $convert:expr) => { written::<$s, $t>(input, range, checked, &mut output, $convert) };
130 (@cast $t:ty) => { write!($t, $t, Ok) };
131 (@keep $t:ty) => { kept_mapped::<$t>(self.input, input, range, checked, &mut output) };
132 (@unsupported) => {{ output.target.fill(range.len()); Err(Error::new(Kind::Domain, "unsupported resolved conversion")) }};
133 }
134 crate::error::checked(|| conversion!(self, checked, tolerance, write))
135 }
136}
137
138#[inline]
139fn converted_item<T: crate::buffer::Element>(value: Result<T, crate::error::Error>) -> (T, bool) {
140 match value { Ok(value) => (value, true), Err(error) => { crate::error::record(error); (T::FILL, false) } }
141}
142
143fn converted_buffer<S: crate::buffer::Cast, T: crate::buffer::Element>(
144 view: crate::buffer::View<'_>, checked: bool, convert: impl Fn(S) -> Result<T, crate::error::Error>
145) -> crate::buffer::Buffer {
146 let (mut values, mut scratch) = (Vec::with_capacity(view.len()), Vec::new());
147 for from in (0..view.len()).step_by(crate::CHUNK) {
148 let part = view.slice(from..(from + crate::CHUNK).min(view.len()));
149 let items = if let Some(items) = S::of(part) { items } else {
150 scratch.clear();
151 if let Err(error) = S::extend_cast(part, part.len(), checked, &mut scratch) {
152 crate::error::record(error); values.resize(values.len() + part.len(), T::FILL); continue;
153 }
154 &scratch
155 };
156 values.extend(items.iter().map(|&x| converted_item(convert(x)).0));
157 }
158 T::buffer(values)
159}
160
161#[inline]
162fn written<S: crate::buffer::Cast, T: crate::buffer::OutputItem>(
163 input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, output: &mut crate::buffer::Output<'_, '_>,
164 convert: impl Fn(S) -> Result<T, crate::error::Error> + Sync
165) -> Result<(), crate::error::Error> {
166 output.write::<T>(range.len(), &|part, slots, work| {
167 let part = part.start + range.start..part.end + range.start;
168 let in_place = input.in_place::<S>(&part);
169 let len = if in_place { part.len().max(1) } else { crate::CHUNK };
170 let mut local = Vec::new();
171 let (mut work, mut copied) = (work, 0);
172 let scratch = match work.as_deref_mut() { Some(work) if !in_place => { let [scratch] = work.inputs::<S, 1>(); scratch }, _ => &mut local };
173 let ok = part.clone().step_by(len).fold(true, |ok, start| {
174 let range = start..(start + len).min(part.end);
175 let n = range.len();
176 let input = match input.part_counted::<S>(range, checked, scratch) {
177 Ok((input, n)) => { copied += n; input }
178 Err(error) => {
179 crate::error::record(error);
180 return unsafe { slots.fill(n, |out| { for slot in out { slot.write(T::FILL); } false }) } & ok;
181 }
182 };
183 slots.pairs(input, crate::Arg::One(()), n, |x, ()| converted_item(convert(x))) & ok
184 });
185 if let Some(work) = work { work.record_copies(copied); }
186 ok
187 })
188}
189
190pub(crate) mod sealed {
191 pub trait Ways: Copy {
193 fn from_f32(x: f32) -> Option<Self>;
194 fn from_f64(x: f64) -> Option<Self>;
195 fn saturated_f32(x: f32) -> Self;
196 fn saturated_f64(x: f64) -> Self;
197 fn to_f32(self) -> f32;
198 fn to_f64(self) -> f64;
199 }
200}
201
202#[inline(always)]
204pub fn int<I: Int, F: Float>(x: F) -> Option<I> { if <F::Working as Core>::WIDE { I::from_f64(x.wide()) } else { I::from_f32(x.wide() as f32) } }
205
206#[inline(always)]
208pub fn saturated<I, S: Saturates<I>>(x: S) -> I { x.saturated() }
209pub trait Saturates<I> { fn saturated(self) -> I; }
211impl<I: Int, F: Float> Saturates<I> for F {
212 #[inline(always)]
213 fn saturated(self) -> I { if <F::Working as Core>::WIDE { I::saturated_f64(self.wide()) } else { I::saturated_f32(self.wide() as f32) } }
214}
215impl<I: Bounded + for<'a> TryFrom<&'a BigInt>> Saturates<I> for &BigInt {
216 fn saturated(self) -> I { I::try_from(self).unwrap_or_else(|_| if self.sign() == Sign::Minus { I::min_value() } else { I::max_value() }) }
217}
218impl<I: Bounded + TryFrom<u64>> Saturates<I> for u64 { fn saturated(self) -> I { I::try_from(self).unwrap_or_else(|_| I::max_value()) } }
219#[inline(always)]
221pub fn floored<I: Int, F: Float>(x: F, t: impl crate::tolerant::Tolerant) -> I { saturated(t.floor(x)) }
222
223#[inline(always)]
225pub fn float<F: Float, I: Int>(i: I) -> F { if <F::Working as Core>::WIDE { F::narrow(i.to_f64()) } else { F::narrow(i.to_f32().into()) } }
226
227pub fn count<F: Float>(out: &mut Vec<F>, start: i64, down: bool, n: usize) {
229 let end = if down { start - n as i64 } else { start + n as i64 };
230 if !<F::Working as Core>::WIDE && i32::try_from(start).is_ok() && i32::try_from(end).is_ok() {
231 let start = start as i32;
232 return if down { out.extend((0..n as i32).map(move |i| float::<F, i32>(start - i))) }
233 else { out.extend((0..n as i32).map(move |i| float::<F, i32>(start + i))) };
234 }
235 if NATIVE_64 {
236 return if down { out.extend((0..n).map(move |i| F::narrow(wide_float(start - i as i64)))) }
237 else { out.extend((0..n).map(move |i| F::narrow(wide_float(start + i as i64)))) };
238 }
239 let step = if down { -1 } else { 1 };
240 let offsets: [f64; 8] = std::array::from_fn(|k| wide_float(step * k as i64));
241 let first = move |j: usize| wide_float(start + step * (8 * j) as i64);
242 unsafe {
243 crate::slice::appended(out, n, |slots| {
244 let (blocks, rest) = slots.as_chunks_mut::<8>();
245 let whole = blocks.len();
246 for (j, block) in blocks.iter_mut().enumerate() { let first = first(j); for k in 0..8 { block[k].write(F::narrow(first + offsets[k])); } }
247 for (slot, &offset) in rest.iter_mut().zip(&offsets) { slot.write(F::narrow(first(whole) + offset)); }
248 })
249 };
250}
251
252const NATIVE_64: bool = cfg!(any(target_arch = "aarch64", target_feature = "avx512dq"));
254
255const SATURATING: bool = !cfg!(target_arch = "x86_64");
257
258pub(crate) const WIDE_LIMIT: f64 = 2251799813685247.0;
260
261#[inline(always)]
263fn wide_float(i: i64) -> f64 { if NATIVE_64 { i as f64 } else { halves(i) } }
264
265#[inline(always)]
267fn halves(i: i64) -> f64 {
268 let high = f64::from_bits(0x4530_0000_0000_0000 | ((i as u64 >> 32) ^ 0x8000_0000)) - f64::from_bits(0x4530_0000_8000_0000);
269 high + f64::from_low(i as u64 & 0xffff_ffff, 0.0)
270}
271
272#[inline(always)]
274pub(crate) fn shifted(x: f64) -> i64 { (x + f64::SHIFT).to_bits().wrapping_sub(f64::SHIFT.to_bits()) as i64 }
275
276#[inline(always)]
278pub(crate) fn unshifted(i: i64) -> f64 { f64::from_bits(f64::SHIFT.to_bits().wrapping_add(i as u64)) - f64::SHIFT }
279
280macro_rules! ints {
282 ($($t:ty: $kind:ident, |$i:ident| $f32:expr, $f64:expr;)+) => {$(
283 impl Int for $t {}
284 impl sealed::Ways for $t {
285 #[inline(always)]
286 fn from_f32(x: f32) -> Option<$t> { ints!(@$kind f32, $t, x) }
287 #[inline(always)]
288 fn from_f64(x: f64) -> Option<$t> { ints!(@$kind f64, $t, x) }
289 #[inline(always)]
290 fn saturated_f32(x: f32) -> $t { ints!(@saturated $kind f32, $t, x) }
291 #[inline(always)]
292 fn saturated_f64(x: f64) -> $t { ints!(@saturated $kind f64, $t, x) }
293 #[inline(always)]
294 fn to_f32(self) -> f32 { let $i = self; $f32 }
295 #[inline(always)]
296 fn to_f64(self) -> f64 { let $i = self; $f64 }
297 }
298 )+};
299 (@narrow $f:ty, $t:ty, $x:ident) => {{
300 let (mid, half) = ((<$t>::MIN as f64 + <$t>::MAX as f64) / 2.0, (<$t>::MAX as f64 - <$t>::MIN as f64 + 1.0) / 2.0);
302 (($x - mid as $f).abs() < half as $f).then(|| {
303 if SATURATING { return $x as i32 as $t }
304 unsafe { $x.to_int_unchecked::<i32>() as $t }
306 })
307 }};
308 (@wide f32, $t:ty, $x:ident) => { Self::from_f64($x.into()) };
309 (@wide f64, $t:ty, $x:ident) => {{
310 let n = if NATIVE_64 { $x as i64 } else { shifted($x) };
311 ($x.abs() <= WIDE_LIMIT).then_some(n).and_then(|n| <$t>::try_from(n).ok())
312 }};
313 (@saturated narrow f32, $t:ty, $x:ident) => { if SATURATING { $x as $t } else { ints!(@clamped f32, $t, $x) } };
314 (@saturated narrow f64, $t:ty, $x:ident) => { if NATIVE_64 { ($x as i64).clamp(<$t>::MIN.into(), <$t>::MAX.into()) as $t } else { ints!(@clamped f64, $t, $x) } };
316 (@saturated wide $f:ty, $t:ty, $x:ident) => { $x as $t };
317 (@clamped $f:ty, $t:ty, $x:ident) => {{
318 let top = <$t>::MAX as $f;
320 let below = if top as i128 > <$t>::MAX as i128 { top.next_down() } else { top };
321 let c = if $x.is_nan() { 0.0 } else { $x.max(<$t>::MIN as $f).min(below) };
322 let i = unsafe { c.to_int_unchecked::<$t>() };
324 if $x >= top { <$t>::MAX } else { i }
325 }};
326}
327
328ints! {
330 i8: narrow, |i| i as f32, i as f64;
331 i16: narrow, |i| i as f32, i as f64;
332 i32: narrow, |i| i as f32, i as f64;
333 u8: narrow, |i| f32::from_low(i.into(), 0.0), f32::from_low(i.into(), 0.0).into();
334 u16: narrow, |i| f32::from_low(i.into(), 0.0), f32::from_low(i.into(), 0.0).into();
335 u32: wide, |i| f64::from_low(i.into(), 0.0) as f32, f64::from_low(i.into(), 0.0);
336 i64: wide, |i| wide_float(i) as f32, wide_float(i);
337}
338
339fn kept_mapped<T: crate::buffer::OutputItem + crate::buffer::Cast>(
340 dtype: crate::buffer::DType, input: crate::agreement::Read<'_>, range: std::ops::Range<usize>, checked: bool, output: &mut crate::buffer::Output<'_, '_>
341) -> Result<(), crate::error::Error> {
342 output.write::<T>(range.len(), &|part, slots, work| {
343 let mut local = crate::buffer::Buffer::with_capacity(dtype, 0);
344 let scratch = match work { Some(work) => work.buffer(dtype), None => &mut local };
345 match input.part_view(part.start + range.start..part.end + range.start, dtype, checked, scratch) {
346 Ok(values) => unsafe { slots.fill(part.len(), |out| {
347 let mut good = true;
348 for (i, slot) in out.iter_mut().enumerate() {
349 slot.write(match crate::buffer::kept_as::<T>(values, i) {
350 Ok(value) => value, Err(error) => { crate::error::record(error); good = false; T::FILL }
351 });
352 }
353 good
354 }) },
355 Err(error) => {
356 crate::error::record(error);
357 unsafe { slots.fill(part.len(), |out| { for slot in out { slot.write(T::FILL); } false }) }
358 }
359 }
360 })
361}