rustymath_core/

slice.rs

//! Slice forms of rustymath's functions. Each function object, such as [`Pow`], has a scalar form `one` and a slice form `many`. Both run code that rustymath compiles once for each item type, inside its own crate. A caller's build settings can't change that code, and rustymath's speed test times what callers run. An edit to a function's body rebuilds only rustymath.
use crate::float::Float;
use num_complex::Complex;
use std::mem::MaybeUninit;

/// How many items a caller handles at a time where it converts or expands arguments into buffers for a slice form. Buffers of this many items stay in cache.
pub const CHUNK: usize = 4096;

/// How many independent running values a loop keeps, each of `size` bytes: a fold's partial results, or a block of items under Horner's rule. 32 measured faster on the Mac than 8 or 16 for items of up to 8 bytes. A 16-byte item, such as a `Complex<f64>`, keeps 16, because 32 fill every vector register, and a complex sum then took 1.7 times as long. In wasm, LLVM leaves 32 values of 8 bytes, or 16 of 16 bytes, in linear memory, where each step loads and stores them. It keeps half as many in locals.
pub const fn running(size: usize) -> usize {
    let wasm = cfg!(target_arch = "wasm32");
    if size >= 16 { if wasm { 8 } else { 16 } } else if wasm && size == 8 { 16 } else { 32 }
}

/// One argument of a slice function: one item that every result pairs with, or one item for each result.
#[derive(Clone, Copy, Debug)]
pub enum Arg<'a, T> { One(T), Many(&'a [T]) }

impl<'a, T: Copy> Arg<'a, T> {
    /// The argument that `items` gives: its only item for every result when it holds one, otherwise one item for each result.
    pub fn of(items: &'a [T]) -> Self { if let [x] = *items { Arg::One(x) } else { Arg::Many(items) } }
    /// Item `i` of this argument, or its one item.
    #[inline(always)]
    pub fn nth(self, i: usize) -> T { match self { Arg::One(v) => v, Arg::Many(s) => s[i] } }
    /// The one item of an argument that holds one for every result.
    #[inline(always)]
    pub fn one(self) -> Option<T> { match self { Arg::One(v) => Some(v), Arg::Many(_) => None } }
    /// Items `start..start + n` of this argument, each converted by `f`. A `Many` argument converts into `buf`.
    #[inline]
    pub fn converted<'b, A>(self, start: usize, n: usize, buf: &'b mut Vec<A>, f: impl Fn(T) -> A) -> Arg<'b, A> {
        match self.part(start, n) {
            Arg::One(v) => Arg::One(f(v)),
            Arg::Many(items) => {
                buf.clear();
                buf.extend(items.iter().map(|&v| f(v)));
                Arg::Many(buf)
            }
        }
    }

    /// Items `start..start + n` of this argument. A `One` argument stays as it is.
    #[inline(always)]
    pub fn part(self, start: usize, n: usize) -> Self { match self { Arg::One(v) => Arg::One(v), Arg::Many(s) => Arg::Many(&s[start..start + n]) } }
}

/// A type that function objects take or give: a float, a complex number, an integer, a Boolean, a character or an ordering. Pieces on several threads read and write items.
pub trait KernelItem: Copy + Send + Sync {
    /// Whether the type has NaN, as floats and complex numbers do.
    const NAN: bool;
    /// Whether this item is NaN, or for a complex number has a NaN part. An integer or Boolean never is.
    fn has_nan(self) -> bool;
}

macro_rules! nan_items { ($($t:ty),+) => {$( impl KernelItem for $t { const NAN: bool = true; #[inline(always)] fn has_nan(self) -> bool { self.is_nan() } } )+} }
nan_items!(f32, f64, half::f16, Complex<f32>, Complex<f64>);
macro_rules! exact_items { ($($t:ty),+) => {$( impl KernelItem for $t { const NAN: bool = false; #[inline(always)] fn has_nan(self) -> bool { false } } )+} }
exact_items!(u8, i16, i32, i64, bool, char, std::cmp::Ordering);

/// The results of a function object of `N` items of type `T`: their type and whether any can fail. [`Unary`], [`Binary`] and [`Ternary`] each require it, so code that takes a function object of any arity reads these through one bound.
pub trait Results<T, const N: usize> {
    /// The type of each result.
    type Out: KernelItem;
    /// Whether no result fails: `checked` always gives `Some`, and `many` and `many_into` always give true. A total float function gives NaN from arguments that aren't NaN only where NaN is its value, as [`Subtract`](crate::Subtract) gives for ∞ - ∞ and [`Sin`](crate::Sin) for ±∞.
    const TOTAL: bool;
}

/// Native scalar and slice interfaces. The slice body initializes every output slot even when an item fails.
macro_rules! arities {
    ($($(#[$doc:meta])* $name:ident<$n:literal>($($x:ident),+);)+) => {$(
        $(#[$doc])*
        ///
        /// # Safety
        /// The slice body initializes every slot in its output before returning.
        pub unsafe trait $name<T>: Results<T, $n> + Sync {
            fn one(&self, $($x: T),+) -> Self::Out;
            fn judged(&self, $($x: T),+) -> (Self::Out, bool);
            fn slice(&self, args: [Arg<'_, T>; $n], n: usize, out: &mut [MaybeUninit<Self::Out>]) -> bool;
            #[inline]
            fn checked(&self, $($x: T),+) -> Option<Self::Out> { let (r, ok) = self.judged($($x),+); ok.then_some(r) }
            fn many(&self, $($x: Arg<'_, T>,)+ n: usize, out: &mut Vec<Self::Out>) -> bool where T: Copy + Sync {
                unsafe { appended(out, n, |slots| crate::parallel::split(slots, CHUNK, &|start, slots| {
                    crate::parallel::in_chunks(start, slots, |at, part| self.slice([$($x.part(at, part.len())),+], part.len(), part))
                })) }
            }
            fn many_into(&self, $($x: Arg<'_, T>,)+ n: usize, out: &mut crate::parallel::Slots<'_, Self::Out>) -> bool where T: Copy {
                // SAFETY: the unsafe object trait promises that `slice` initializes every output.
                unsafe { out.fill(n, |slots| crate::parallel::in_chunks(0, slots, |at, part| self.slice([$($x.part(at, part.len())),+], part.len(), part))) }
            }
        }
    )+};
}

arities! {
    /// A function of one item as a value, with a scalar form and a slice form. rustymath's function objects, such as [`Exp`](crate::Exp), implement it at each item type they take: every [`Float`] type for a float function, and complex numbers, integers or Booleans for others. `Out` is the type of each result, which is the item type unless the function gives another, as [`Abs`](crate::Abs) gives the real magnitude of a complex number. Its methods take `&self`, so a function object can hold settings.
    Unary<1>(x);
    /// A function of two items as a value, as [`Unary`] is for one.
    Binary<2>(x, y);
    /// A function of three items as a value, as [`Unary`] is for one.
    Ternary<3>(x, y, z);
}

/// `K` with its two arguments in the other order, as a [`Binary`] at each item type where `K` is one: `Flipped(k).one(x, y)` is `k.one(y, x)`, and likewise for `many` and `checked`.
#[derive(Clone, Copy, Debug)]
pub struct Flipped<K>(pub K);

impl<T, K: Results<T, 2>> Results<T, 2> for Flipped<K> {
    type Out = K::Out;
    const TOTAL: bool = K::TOTAL;
}
unsafe impl<T, K: Binary<T>> Binary<T> for Flipped<K> {
    fn one(&self, x: T, y: T) -> K::Out { self.0.one(y, x) }
    fn judged(&self, x: T, y: T) -> (K::Out, bool) { self.0.judged(y, x) }
    fn slice(&self, [x, y]: [Arg<'_, T>; 2], n: usize, out: &mut [MaybeUninit<K::Out>]) -> bool { self.0.slice([y, x], n, out) }
}

/// `K` with no result failing. Each result is `K`'s, wrapped to its type, as static kinds and NumPy keep integer results. LLVM removes `K`'s test of whether a result fits, because the loop never reads it. The adapter suits an object whose scalar form is its whole computation, as integer arithmetic's is.
#[derive(Clone, Copy, Debug)]
pub struct Wrapped<K>(pub K);

impl<T, K: Results<T, N>, const N: usize> Results<T, N> for Wrapped<K> {
    type Out = K::Out;
    const TOTAL: bool = true;
}
macro_rules! wrapped {
    ($arity:ident, $n:literal, $($x:ident),+) => {
        unsafe impl<T: KernelItem, K: $arity<T>> $arity<T> for Wrapped<K> {
            fn one(&self, $($x: T),+) -> K::Out { self.0.one($($x),+) }
            fn judged(&self, $($x: T),+) -> (K::Out, bool) { (self.0.one($($x),+), true) }
            fn slice(&self, x: [Arg<'_, T>; $n], n: usize, out: &mut [MaybeUninit<K::Out>]) -> bool {
                each(x, n, out, #[inline(always)] |[$($x),+]| Total(self.0.one($($x),+)))
            }
        }
    };
}
wrapped!(Unary, 1, x);
wrapped!(Binary, 2, x, y);
wrapped!(Ternary, 3, x, y, z);

/// Whether `r`, the result for arguments `args`, keeps the rule that a NaN result needs a NaN argument.
#[inline(always)]
pub(crate) fn kept<O: KernelItem, E: KernelItem, const N: usize>(r: O, args: [E; N]) -> bool { !r.has_nan() | args.iter().any(|a| a.has_nan()) }

/// What a function object's item body gives: an output, which fails where it is NaN and no argument is, an `Option` of one, which fails where it is `None`, a wrapped output paired with whether it's exact, which fails where it isn't, or a [`Total`] output, which never fails. An integer power, whose result can overflow, gives a wrapped output. An integer function with no wrapped result to give, such as a binomial coefficient, gives an `Option`. An output that is never NaN, such as `bool`, fails only through `None` or an inexact pair.
pub(crate) trait Outcome<O: KernelItem>: Sized {
    /// Whether an output that `split` fails can still succeed, as a NaN output does where an argument is NaN. Only a plain output can.
    const RESCUE: bool;
    /// Whether no output fails: a [`Total`] output, or a plain output of a type without NaN.
    const TOTAL: bool;
    /// The output, as a scalar form gives it. `None` panics, because a scalar form has no other way to report it.
    fn value(self) -> O;
    /// The output, and whether it succeeded, judged by the output alone, so that a NaN output fails. That costs one test for each item, where testing each argument too would cost one more for each argument.
    fn split(self) -> (O, bool);
    /// The output, and whether it succeeded for arguments `args`.
    #[inline(always)]
    fn judged<E: KernelItem, const N: usize>(self, args: [E; N]) -> (O, bool) { let (r, ok) = self.split(); (r, ok || (Self::RESCUE && kept(r, args))) }
}

impl<O: KernelItem> Outcome<O> for O {
    const RESCUE: bool = true;
    const TOTAL: bool = !O::NAN;
    #[inline(always)]
    fn value(self) -> O { self }
    #[inline(always)]
    fn split(self) -> (O, bool) { (self, !self.has_nan()) }
}

impl<O: KernelItem + Default> Outcome<O> for Option<O> {
    const RESCUE: bool = false;
    const TOTAL: bool = false;
    #[inline(always)]
    fn value(self) -> O { self.expect("the result doesn't fit the output type") }
    #[inline(always)]
    fn split(self) -> (O, bool) { (self.unwrap_or_default(), self.is_some()) }
}

/// An output wrapped to its type, and whether it's the exact result, as an integer power gives it. A result that isn't exact fails, and its slot holds the wrapped output. That output is NumPy's result for fixed-width integers.
impl<O: KernelItem> Outcome<O> for (O, bool) {
    const RESCUE: bool = false;
    const TOTAL: bool = false;
    #[inline(always)]
    fn value(self) -> O { self.0 }
    #[inline(always)]
    fn split(self) -> (O, bool) { self }
}

/// The output of an item body that never fails: one that never gives NaN from arguments that aren't NaN, such as `exp`'s, or one whose NaN is the function's value, such as that of `∞ - ∞`. The slice loop tests none of them.
#[derive(Clone, Copy)]
pub(crate) struct Total<O>(pub O);

impl<O: KernelItem> Outcome<O> for Total<O> {
    const RESCUE: bool = false;
    const TOTAL: bool = true;
    #[inline(always)]
    fn value(self) -> O { self.0 }
    #[inline(always)]
    fn split(self) -> (O, bool) { (self.0, true) }
}

/// Whether an item body never fails, as its outcome type `R` says. `forms!` passes the body as a closure, which names that type without running it.
pub(crate) const fn total<K, E, O: KernelItem, R: Outcome<O>, const N: usize>(_: fn(&K, [E; N]) -> R) -> bool { R::TOTAL }

/// The items of `x` that result `i` takes.
#[inline(always)]
fn items<E: Copy, const N: usize>(x: &[Arg<'_, E>; N], i: usize) -> [E; N] { std::array::from_fn(|k| x[k].nth(i)) }

/// Writes `f` for each of `n` results into `out`, for arguments `x`, and gives whether `f` said `true` for every result. One and two arguments run through [`pairs_into`], and three through `triples_into`, whose loops vectorize.
#[inline(always)]
pub fn mapped<E: Copy, O, const N: usize>(x: &[Arg<'_, E>; N], n: usize, out: &mut [MaybeUninit<O>], f: impl Fn([E; N]) -> (O, bool)) -> bool {
    if const { N == 1 } { pairs_into(x[0], Arg::One(()), n, out, #[inline(always)] |a, ()| f([a; N])) }
    else if const { N == 2 } {
        pairs_into(x[0], x[1], n, out, #[inline(always)] |a, b| f(std::array::from_fn(|k| if k == 0 { a } else { b })))
    } else if const { N == 3 } {
        triples_into(x[0], x[1], x[2], n, out, #[inline(always)] |a, b, c| {
            f(std::array::from_fn(|k| match k { 0 => a, 1 => b, _ => c }))
        })
    } else { unreachable!("function objects take one to three arguments") }
}

/// Writes `f` for each of `n` results into `out`, for arguments `x`, and gives whether every result succeeded, as [`Outcome`] says. Slice forms call it through [`each!`]. The loop judges each output alone, and a [`Total`] output not at all. Only where some output is NaN does a second pass look at the arguments of the NaN outputs, which is rare.
#[inline(always)]
pub(crate) fn each<E: KernelItem, O: KernelItem, R: Outcome<O>, const N: usize>(
    x: [Arg<'_, E>; N], n: usize, out: &mut [MaybeUninit<O>], f: impl Fn([E; N]) -> R) -> bool {
    mapped(
        &x, n, out, #[inline(always)] |a| f(a).split()
    ) || (R::RESCUE && unsafe { written(&mut out[..n]) }.iter().enumerate().all(|(i, &r)| kept(r, items(&x, i))))
}

/// A rare form for `each_usual` that no block takes, for a function with no rare form.
pub(crate) fn none<E, O>() -> (impl Fn(E) -> bool, impl Fn(E) -> O) { (|_| false, |_| unreachable!("`none` accepts no item")) }

/// Writes a form of each item of `xs` into `out`, a block of `BLOCK` items at a time, and gives whether `mixed` said `true` for every block it wrote. `uniform` tests a whole block and gives whether every item of it takes `first`'s form, then whether every item takes `second`'s. A block that neither covers, and the last part block, goes to `mixed`. A test for each block keeps rare items, such as a function's special values, out of the loop over a uniform block. Each caller's test suits its forms: `each_usual` folds two tests over the items, and `erfc` folds a minimum and a maximum of their magnitudes.
#[inline(always)]
pub(crate) fn blocks<E: Copy, O, const BLOCK: usize>(
    xs: &[E], out: &mut [MaybeUninit<O>], uniform: impl Fn(&[E; BLOCK]) -> (bool, bool), first: impl Fn(E) -> O, second: impl Fn(E) -> O,
    mut mixed: impl FnMut(&[E], &mut [MaybeUninit<O>]) -> bool) -> bool {
    let (blocks, rest) = xs.as_chunks::<BLOCK>();
    let (slots, last) = out[..xs.len()].split_at_mut(blocks.len() * BLOCK);
    let mut ok = true;
    for (b, o) in blocks.iter().zip(slots.as_chunks_mut::<BLOCK>().0) {
        let (one, two) = uniform(b);
        if one { for k in 0..BLOCK { o[k].write(first(b[k])); } } else if two { for k in 0..BLOCK { o[k].write(second(b[k])); } } else { ok &= mixed(b, o) }
    }
    ok & mixed(rest, last)
}

/// [`each`] for one argument, through [`blocks`] of 64 items. A block whose every item `usual`'s test accepts writes `usual`'s form of each item. Failing that, a block whose every item `rare`'s test accepts writes `rare`'s form, as `exp` writes 0 for a block below its underflow bound. Any other block writes `body` of each item, judged as `each` judges it. The two forms never fail.
#[inline(always)]
pub(crate) fn each_usual<E: KernelItem, O: KernelItem, R: Outcome<O>>(
    x: Arg<'_, E>, n: usize, out: &mut [MaybeUninit<O>], usual: (impl Fn(E) -> bool, impl Fn(E) -> O), rare: (impl Fn(E) -> bool, impl Fn(E) -> O),
    body: impl Fn(E) -> R) -> bool {
    let Arg::Many(xs) = x else { return each([x], n, out, #[inline(always)] |[x]| body(x)); };
    // With blocks of 16, f64's test and the fast form's constants outgrew AArch64's vector registers, and the loop rebuilt the constants for each block.
    blocks::<_, _, 64>(
        &xs[..n], out, #[inline(always)] |b| b.iter().fold((true, true), |(one, two), &v| (one & (usual.0)(v), two & (rare.0)(v))), usual.1, rare.1,
        #[inline(always)] |b, o| {
            each([Arg::Many(b)], b.len(), o, #[inline(always)] |[x]| body(x))
        })
}

/// Declares a function object of one argument whose slice form runs `each_usual`. `$usual` says whether an item needs none of the function's special values, `$fast` computes such an item, and `$name` is the full function. An optional `$rare => $value` gives the form for a block whose items all take one rare value, as `exp` gives 0 below its underflow bound. A trailing `; total` wraps the full function's results in [`Total`].
macro_rules! usual_object {
    ($(#[doc = $doc:literal])* $object:ident: $name:ident, $usual:expr => $fast:expr $(, $rare:expr => $value:expr)? $(; $outcome:ident)?) => {
        $(#[doc = $doc])*
        #[derive(Clone, Copy, Debug)]
        pub struct $object;
        forms!($object at real: |_, [x]| usual_object!(@outcome [$($outcome)?], $name::<_, { crate::FUSED }>(x)),
            many |_, [x], n, out| crate::slice::each_usual(x, n, out, ($usual, $fast), usual_object!(@rare $($rare, $value)?),
                #[inline(always)] |x| usual_object!(@outcome [$($outcome)?], $name::<_, { crate::FUSED }>(x))));
    };
    (@rare) => { crate::slice::none() };
    (@rare $rare:expr, $value:expr) => { ($rare, $value) };
    (@outcome [], $e:expr) => { $e };
    (@outcome [total], $e:expr) => { crate::slice::Total($e) };
}

/// [`each`] with the item body marked `#[inline(always)]`. Every slice form runs its loop this way: `forms!`'s default slice form, and a custom slice form once it has read its settings. LLVM leaves a large unmarked body out of line, and its call stops the loop vectorizing.
macro_rules! each {
    ($x:expr, $n:expr, $out:expr, |$args:pat_param| $body:expr $(,)?) => {
        crate::slice::each(
            $x, $n, $out, #[inline(always)] |$args| $body,
        )
    };
}

/// Writes a function object's forms, compiled inside rustymath. The list names the item types, such as `Complex<f32>` or `i64`. `real` stands for `f32` and `f64`, and also writes an `f16` form, which runs the `f32` form and narrows its outputs. `=> T` makes the output type `T` in place of the item type: for each listed type, or after `real` for every float type. The object is the first argument of the item body, and its items the bracketed ones. The body gives an output, an `Option` of one, or a [`Total`] output, as [`Outcome`] says. A `many` body replaces the default slice form, which runs the item body through [`each!`]. Its arguments are the object, the bracketed `Arg`s, the number of results, and the slots it writes them into. Attributes before the object, such as `#[inline]`, go on its scalar forms, `one` and `judged`, so that a caller in another crate can inline a small form into its own loop over items.
///
/// A function object is a concrete type, never a generic one, so that its forms compile here. Settings that choose a loop, such as whether a tolerance is 0, are read once per call inside `many`, which then runs `each!`.
///
/// ```text
/// forms!(Choose at real: |_, [x, y, m]| Total(if m != 0.0 { x } else { y }));
/// forms!(Choose at bool: |_, [x, y, m]| (m & x) | (!m & y));
/// forms!(Gcd at i64: |_, [x, y]| gcd(x, y));
/// forms!(Less at real => bool: |k, [x, y]| k.0.less(x, y),
///     many |k, [x, y], n, out| with_tolerance!(k.0, |t| each!([x, y], n, out, |[x, y]| t.less(x, y))));
/// ```
macro_rules! forms {
    ($(#[$($m:tt)+])* $object:ident at real $(=> $o:ty)? : |$k:pat_param, $args:tt| $body:expr $(, many |$mk:pat_param, $margs:tt, $n:ident, $out:ident| $many:expr)?) => {
        forms!(@types [$(#[$($m)+])*], $object, $k, $args, $body, [$((|$mk, $margs, $n, $out| $many))?], [f32 $(=> $o)?, f64 $(=> $o)?]);
        forms!(@half [$(#[$($m)+])*], $object, forms!(@out half::f16 $(, $o)?), forms!(@narrow $($o)?), $args);
    };
    ($(#[$($m:tt)+])* $object:ident at $($e:ty $(=> $o:ty)?),+ : |$k:pat_param, $args:tt| $body:expr $(, many |$mk:pat_param, $margs:tt, $n:ident, $out:ident| $many:expr)?) => {
        forms!(@types [$(#[$($m)+])*], $object, $k, $args, $body, [$((|$mk, $margs, $n, $out| $many))?], [$($e $(=> $o)?),+]);
    };
    // A generic type, such as `Indicator<Less>`, goes in brackets, because a type can't be followed by `at`.
    ($(#[$($m:tt)+])* [$object:ty] at $($e:ty $(=> $o:ty)?),+ : |$k:pat_param, $args:tt| $body:expr $(, many |$mk:pat_param, $margs:tt, $n:ident, $out:ident| $many:expr)?) => {
        forms!(@types [$(#[$($m)+])*], $object, $k, $args, $body, [$((|$mk, $margs, $n, $out| $many))?], [$($e $(=> $o)?),+]);
    };
    // One type at a time, so that the optional `many` group, which repeats on its own, needn't sit inside the types' repetition.
    (@types $attrs:tt, $object:ty, $k:pat_param, $args:tt, $body:expr, $many:tt, [$e:ty $(=> $o:ty)? $(, $($rest:tt)+)?]) => {
        forms!(@concrete $attrs, $object, $e, forms!(@out $e $(, $o)?), $k, $args, $body, $many);
        $(forms!(@types $attrs, $object, $k, $args, $body, $many, [$($rest)+]);)?
    };
    (@concrete $attrs:tt, $object:ty, $e:ty, $o:ty, $k:pat_param, [$x:ident], $body:expr, $many:tt) => {
        forms!(@impl $attrs, $object, Unary, 1, $e, $o, $k, [$x], $body, $many);
    };
    (@concrete $attrs:tt, $object:ty, $e:ty, $o:ty, $k:pat_param, [$x:ident, $y:ident], $body:expr, $many:tt) => {
        forms!(@impl $attrs, $object, Binary, 2, $e, $o, $k, [$x, $y], $body, $many);
    };
    (@concrete $attrs:tt, $object:ty, $e:ty, $o:ty, $k:pat_param, [$x:ident, $y:ident, $z:ident], $body:expr, $many:tt) => {
        forms!(@impl $attrs, $object, Ternary, 3, $e, $o, $k, [$x, $y, $z], $body, $many);
    };
    (@impl [$(#[$($m:tt)+])*], $object:ty, $arity:ident, $arity_n:literal, $e:ty, $o:ty, $k:pat_param, [$($x:ident),+], $body:expr, $many:tt) => {
        impl crate::slice::Results<$e, $arity_n> for $object {
            type Out = $o;
            const TOTAL: bool = crate::slice::total::<Self, $e, $o, _, $arity_n>(|$k, [$($x),+]| $body);
        }
        unsafe impl crate::slice::$arity<$e> for $object {
            $(#[$($m)+])*
            fn one(&self, $($x: $e),+) -> $o {
                let $k = self;
                crate::slice::Outcome::<$o>::value($body)
            }
            $(#[$($m)+])*
            fn judged(&self, $($x: $e),+) -> ($o, bool) {
                let $k = self;
                crate::slice::Outcome::<$o>::judged($body, [$($x),+])
            }
            forms!(@slice $e => $o, $k, [$($x),+], $body, $many);
        }
    };
    (@slice $e:ty => $o:ty, $k:pat_param, [$($x:ident),+], $body:expr, []) => {
        fn slice(&self, args: [crate::Arg<'_, $e>; forms!(@n $($x),+)], n: usize, out: &mut [std::mem::MaybeUninit<$o>]) -> bool {
            let $k = self;
            each!(args, n, out, |[$($x),+]| $body)
        }
    };
    (@slice $e:ty => $o:ty, $k:pat_param, $args:tt, $body:expr, [(|$mk:pat_param, [$($mx:ident),+], $n:ident, $out:ident| $many:expr)]) => {
        fn slice(&self, [$($mx),+]: [crate::Arg<'_, $e>; forms!(@n $($mx),+)], $n: usize, $out: &mut [std::mem::MaybeUninit<$o>]) -> bool {
            let $mk = self;
            $many
        }
    };
    (@half $attrs:tt, $object:ty, $o:ty, $narrow:expr, [$x:ident]) => { forms!(@half_impl $attrs, $object, Unary, 1, $o, $narrow, [$x]); };
    (@half $attrs:tt, $object:ty, $o:ty, $narrow:expr, [$x:ident, $y:ident]) => { forms!(@half_impl $attrs, $object, Binary, 2, $o, $narrow, [$x, $y]); };
    (@half $attrs:tt, $object:ty, $o:ty, $narrow:expr, [$x:ident, $y:ident, $z:ident]) => { forms!(@half_impl $attrs, $object, Ternary, 3, $o, $narrow, [$x, $y, $z]); };
    (@half_impl [$(#[$($m:tt)+])*], $object:ty, $arity:ident, $arity_n:literal, $o:ty, $narrow:expr, [$($x:ident),+]) => {
        impl crate::slice::Results<half::f16, $arity_n> for $object {
            type Out = $o;
            const TOTAL: bool = <Self as crate::slice::Results<f32, $arity_n>>::TOTAL;
        }
        unsafe impl crate::slice::$arity<half::f16> for $object {
            $(#[$($m)+])*
            fn one(&self, $($x: half::f16),+) -> $o { ($narrow)(crate::slice::$arity::<f32>::one(self, $($x.to_f32()),+)) }
            $(#[$($m)+])*
            fn judged(&self, $($x: half::f16),+) -> ($o, bool) {
                let (r, ok) = crate::slice::$arity::<f32>::judged(self, $($x.to_f32()),+);
                (($narrow)(r), ok)
            }
            fn slice(&self, x: [crate::Arg<'_, half::f16>; $arity_n], n: usize, out: &mut [std::mem::MaybeUninit<$o>]) -> bool {
                crate::slice::working(x, n, out, |x, m, o| crate::slice::$arity::<f32>::slice(self, x, m, o), $narrow)
            }
        }
    };
    (@out $e:ty) => { $e };
    (@out $e:ty, $o:ty) => { $o };
    (@narrow) => { half::f16::from_f32 };
    (@narrow $o:ty) => { std::convert::identity };
    (@n $($x:ident),+) => { [$(stringify!($x)),+].len() };
}

/// Runs `$body` for each item `$i` of a result of `$n` items, with `$a` and `$b` the items of `$x` and `$y` that pair with it. Each pairing of `One` and `Many` has a loop of its own. The loops index slices of the result's length with no iterator adapters, and vectorize at any build settings.
macro_rules! each_pair {
    ($x:expr, $y:expr, $n:expr, |$i:ident, $a:ident, $b:ident| $body:expr) => {{
        let n = $n;
        match ($x, $y) {
            (Arg::One($a), Arg::One($b)) => {
                for $i in 0..n {
                    $body
                }
            }
            (Arg::One($a), Arg::Many(ys)) => {
                let ys = &ys[..n];
                for $i in 0..n {
                    let $b = ys[$i];
                    $body
                }
            }
            (Arg::Many(xs), Arg::One($b)) => {
                let xs = &xs[..n];
                for $i in 0..n {
                    let $a = xs[$i];
                    $body
                }
            }
            (Arg::Many(xs), Arg::Many(ys)) => {
                let (xs, ys) = (&xs[..n], &ys[..n]);
                for $i in 0..n {
                    let ($a, $b) = (xs[$i], ys[$i]);
                    $body
                }
            }
        }
    }};
}

/// Runs `$body` for each item `$i` of a result of `$n` items, with `$a`, `$b` and `$c` the items of `$x`, `$y` and `$z` that go with it, as [`each_pair!`] does for two arguments. Each pairing of `One` and `Many` has a loop of its own.
macro_rules! each_triple {
    ($x:expr, $y:expr, $z:expr, $n:expr, |$i:ident, $a:ident, $b:ident, $c:ident| $body:expr) => {{
        let n = $n;
        match $x {
            Arg::One($a) => each_pair!($y, $z, n, |$i, $b, $c| $body),
            Arg::Many(firsts) => {
                let firsts = &firsts[..n];
                each_pair!($y, $z, n, |$i, $b, $c| {
                    let $a = firsts[$i];
                    $body
                })
            }
        }
    }};
}

/// Appends `f` on each of `n` pairs of items to `out`, and gives whether `f` said `true` for every pair. An `Arg::One` pairs with every item of the other argument. The loops write straight into `out`'s spare capacity, which is never zeroed first. This is the loop that rustymath's slice forms run, and a caller's own kernels can run it too, as basedpl's arithmetic does. A long call splits across threads, as [`parallel`](crate::parallel) describes.
#[inline(never)]
pub fn pairs<X: Copy + Sync, Y: Copy + Sync, B: Send>(x: Arg<'_, X>, y: Arg<'_, Y>, n: usize, out: &mut Vec<B>, f: impl Fn(X, Y) -> (B, bool) + Sync) -> bool {
    unsafe {
        appended(out, n, |slots| {
            crate::parallel::split(slots, CHUNK, &|start, slots| {
                pairs_into(x.part(start, slots.len()), y.part(start, slots.len()), slots.len(), slots, |a, b| f(a, b))
            })
        })
    }
}

/// [`pairs`], writing into the `n` slots of `out` in place of appending to a vector. It runs [`triples_into`]'s loops with a unit first argument, which go straight to [`each_pair!`]'s.
#[inline]
pub(crate) fn pairs_into<X: Copy, Y: Copy, B>(x: Arg<'_, X>, y: Arg<'_, Y>, n: usize, out: &mut [MaybeUninit<B>], f: impl Fn(X, Y) -> (B, bool)) -> bool {
    triples_into(Arg::One(()), x, y, n, out, #[inline(always)] |(), a, b| f(a, b))
}

/// [`pairs_into`] for three arguments, which `mapped` runs for a function object of three arguments.
#[inline]
pub(crate) fn triples_into<X: Copy, Y: Copy, Z: Copy, B>(
    x: Arg<'_, X>, y: Arg<'_, Y>, z: Arg<'_, Z>, n: usize, out: &mut [MaybeUninit<B>], f: impl Fn(X, Y, Z) -> (B, bool)) -> bool {
    let (out, mut ok) = (&mut out[..n], true);
    each_triple!(x, y, z, n, |i, a, b, c| {
        let (r, good) = f(a, b, c);
        out[i].write(r);
        ok &= good;
    });
    ok
}

/// Whether `valid` holds for every result in `out` with the items of `x` that gave it, paired as [`mapped`] pairs them. These loops only judge: a loop that also computes each result stops vectorizing for some bodies, such as `Ln`'s. Failures gather as bits, with no branch.
#[inline(always)]
pub fn judged<E: Copy, O: Copy, const N: usize>(x: &[Arg<'_, E>; N], out: &[O], valid: impl Fn([E; N], O) -> bool) -> bool {
    let (n, mut bad) = (out.len(), 0u8);
    if const { N == 1 } { each_pair!(x[0], Arg::Many(out), n, |_i, a, r| bad |= u8::from(!valid([a; N], r))); }
    else if const { N == 2 } {
        each_triple!(x[0], x[1], Arg::Many(out), n, |_i, a, b, r| bad |= u8::from(!valid(std::array::from_fn(|k| if k == 0 { a } else { b }), r)));
    } else { for i in 0..n { bad |= u8::from(!valid(items(x, i), out[i])); } }
    bad == 0
}

/// Appends `n` items to `out`, which `write` writes into its spare capacity, never zeroed first, and gives `write`'s result. `write` must write every slot it gets. Each caller's loop does, by its construction.
///
/// # Safety
/// `write` must initialize every slot it receives before returning.
#[inline(always)]
pub unsafe fn appended<B, T>(out: &mut Vec<B>, n: usize, write: impl FnOnce(&mut [MaybeUninit<B>]) -> T) -> T {
    out.reserve(n);
    let result = write(&mut out.spare_capacity_mut()[..n]);
    // SAFETY: `write` wrote all `n` slots.
    unsafe { out.set_len(out.len() + n) }
    result
}

/// `out`'s slots as items. Each caller passes only slots that its loop has just written.
///
/// # Safety
/// Every slot in `out` must hold an initialized `B`.
#[inline(always)]
pub unsafe fn written<B>(out: &mut [MaybeUninit<B>]) -> &mut [B] {
    // SAFETY: the caller has written every slot.
    unsafe { &mut *(std::ptr::from_mut(out) as *mut [B]) }
}

/// Writes `f` for each of `n` results into `out`, for arguments `x`, for a function `f` with a fast form that holds for all but rare items. `fast` computes every item with no branch, and gives `None` for the rare items. The items run in two passes. The first writes NaN for each rare item and judges each result alone, as [`each`] does. The second runs only if a result is NaN. It recomputes each NaN item with `f` and judges it by its arguments too. A rare item costs one more scan of at most one `CHUNK`, the most that `parallel::in_chunks` hands a slice form. Slice forms call it through [`two_pass!`].
#[inline(always)]
pub(crate) fn two_pass<R, O, const N: usize>(
    x: [Arg<'_, R>; N], n: usize, out: &mut [MaybeUninit<O>], fast: impl Fn([R; N]) -> Option<R>, f: impl Fn([R; N]) -> R) -> bool where
    R: crate::Float + crate::buffer::Source<O>, O: crate::Float {
    let out = &mut out[..n];
    let first = mapped(&x, n, out, #[inline(always)] |a| {
        let r = <R as crate::buffer::Source<O>>::read(fast(a).unwrap_or(R::nan()));
        (r, !r.is_nan())
    });
    if first { return true; }
    let mut ok = true;
    for (i, r) in unsafe { written(out) }.iter_mut().enumerate().filter(|(_, r)| r.is_nan()) {
        let a = items(&x, i);
        *r = <R as crate::buffer::Source<O>>::read(f(a));
        ok &= kept(*r, a);
    }
    ok
}

/// `f`, a slice form at `F`'s working type, for narrower floats, as `forms!`'s `f16` forms run it: the items convert to the working type and run, and `narrow` turns each output into the output at `F`.
pub(crate) fn working<F: Float, O: Copy, P, const N: usize>(
    x: [Arg<'_, F>; N], n: usize, out: &mut [MaybeUninit<P>], f: impl Fn([Arg<'_, F::Working>; N], usize, &mut [MaybeUninit<O>]) -> bool,
    narrow: impl Fn(O) -> P) -> bool {
    let mut bufs = [(); N].map(|()| Vec::new());
    let mut next = bufs.each_mut().into_iter();
    unsafe { buffered(n, out, |o| f(x.map(|a| a.converted(0, n, next.next().unwrap(), F::working)), n, o), narrow) }
}

/// `f`, a slice form writing `n` outputs of type `O`, run into a buffer, with `each` turning each output into the `P` that `out` takes. It gives `f`'s result.
///
/// # Safety
/// `f` must initialize every output slot it receives before returning.
pub unsafe fn buffered<O: Copy, P>(n: usize, out: &mut [MaybeUninit<P>], f: impl FnOnce(&mut [MaybeUninit<O>]) -> bool, each: impl Fn(O) -> P) -> bool {
    let mut rs = Vec::new();
    let ok = unsafe { appended(&mut rs, n, f) };
    pairs_into(Arg::Many(&rs), Arg::One(()), n, &mut out[..n], |r, ()| (each(r), true));
    ok
}