use crate::num::arithmetic::mod_mul::{limbs_invert_limb_u64, mod_preinverted_double};
use crate::num::arithmetic::traits::{ModDot, ModIsReduced};
use crate::num::basic::integers::PrimitiveInt;
use crate::num::basic::unsigneds::PrimitiveUnsigned;
use crate::num::logic::traits::LeadingZeros;
use crate::unsigned_vector::UnsignedVector;
pub(crate) struct ModData<T: PrimitiveUnsigned> {
pub(crate) m: T,
inv: u64,
radix: T,
pub(crate) words: u8,
pub(crate) max_terms: usize,
}
impl<T: PrimitiveUnsigned> ModData<T> {
pub(crate) fn new(m: T, terms: usize) -> Self {
assert_ne!(m, T::ZERO);
let words = match T::try_from(terms) {
Ok(terms) => {
let (hi, lo) = T::x_mul_y_to_zz(m - T::ONE, m - T::ONE);
let carry = T::x_mul_y_to_zz(lo, terms).0;
let (top, mid) = T::x_mul_y_to_zz(hi, terms);
let (top, mid) = T::xx_add_yy_to_zz(top, mid, T::ZERO, carry);
if top != T::ZERO {
3
} else if mid != T::ZERO {
2
} else {
1
}
}
Err(_) => 3,
};
let inv = if T::WIDTH == u64::WIDTH {
let m: u64 = m.wrapping_into();
limbs_invert_limb_u64(m << LeadingZeros::leading_zeros(m))
} else {
0
};
Self {
m,
inv,
radix: if T::WIDTH > u64::WIDTH {
m.wrapping_neg() % m
} else {
T::ZERO
},
words,
max_terms: if T::WIDTH > usize::WIDTH {
usize::MAX
} else {
1 << (T::WIDTH - 1)
},
}
}
#[inline]
pub(crate) fn reduce_2(&self, x1: T, x0: T) -> T {
let m = self.m;
if T::WIDTH <= u32::WIDTH {
let x: u64 = x1.wrapping_into();
let x0: u64 = x0.wrapping_into();
let m: u64 = m.wrapping_into();
T::wrapping_from(((x << T::WIDTH) | x0) % m)
} else if T::WIDTH == u64::WIDTH {
T::wrapping_from(mod_preinverted_double::<u64, u128>(
x1.wrapping_into(),
x0.wrapping_into(),
m.wrapping_into(),
self.inv,
))
} else {
let data = T::precompute_mod_mul_data(&m);
(x1 % m)
.mod_mul_precomputed(self.radix, m, &data)
.mod_add(x0 % m, m)
}
}
#[inline]
pub(crate) fn reduce(&self, x2: T, x1: T, x0: T) -> T {
self.reduce_2(self.reduce_2(x2, x1), x0)
}
#[inline]
pub(crate) fn reduce_sum(&self, (x2, x1, x0): (T, T, T)) -> T {
match self.words {
1 => x0 % self.m,
2 => self.reduce_2(x1, x0),
_ => self.reduce(x2, x1, x0),
}
}
}
#[inline]
pub(crate) fn accumulate<T: PrimitiveUnsigned>(acc: &mut (T, T, T), x: T, y: T) {
let (hi, lo) = T::x_mul_y_to_zz(x, y);
let (a2, a1, a0) = *acc;
*acc = T::xxx_add_yyy_to_zzz(a2, a1, a0, T::ZERO, hi, lo);
}
#[inline]
pub(crate) fn column_sum<T: PrimitiveUnsigned>(xs: &[T], ys: &[T], d: &ModData<T>) -> (T, T, T) {
match d.words {
1 => {
let mut sum = T::ZERO;
for (&x, &y) in xs.iter().zip(ys.iter().rev()) {
sum.wrapping_add_assign(x.wrapping_mul(y));
}
(T::ZERO, T::ZERO, sum)
}
2 => {
let (mut hi, mut lo) = (T::ZERO, T::ZERO);
for (&x, &y) in xs.iter().zip(ys.iter().rev()) {
let (p_hi, p_lo) = T::x_mul_y_to_zz(x, y);
(hi, lo) = T::xx_add_yy_to_zz(hi, lo, p_hi, p_lo);
}
(T::ZERO, hi, lo)
}
_ => {
let mut acc = (T::ZERO, T::ZERO, T::ZERO);
for (i, (x_chunk, y_chunk)) in xs
.chunks(d.max_terms)
.zip(ys.rchunks(d.max_terms))
.enumerate()
{
if i != 0 {
acc = (T::ZERO, T::ZERO, d.reduce(acc.0, acc.1, acc.2));
}
for (&x, &y) in x_chunk.iter().zip(y_chunk.iter().rev()) {
accumulate(&mut acc, x, y);
}
}
acc
}
}
}
#[inline]
pub(crate) fn dot_sum<T: PrimitiveUnsigned>(xs: &[T], ys: &[T], d: &ModData<T>) -> (T, T, T) {
match d.words {
1 => {
let mut sum = T::ZERO;
for (&x, &y) in xs.iter().zip(ys) {
sum.wrapping_add_assign(x.wrapping_mul(y));
}
(T::ZERO, T::ZERO, sum)
}
2 => {
let (mut hi, mut lo) = (T::ZERO, T::ZERO);
for (&x, &y) in xs.iter().zip(ys) {
let (p_hi, p_lo) = T::x_mul_y_to_zz(x, y);
(hi, lo) = T::xx_add_yy_to_zz(hi, lo, p_hi, p_lo);
}
(T::ZERO, hi, lo)
}
_ => {
let mut acc = (T::ZERO, T::ZERO, T::ZERO);
for (i, (x_chunk, y_chunk)) in xs
.chunks(d.max_terms)
.zip(ys.chunks(d.max_terms))
.enumerate()
{
if i != 0 {
acc = (T::ZERO, T::ZERO, d.reduce(acc.0, acc.1, acc.2));
}
for (&x, &y) in x_chunk.iter().zip(y_chunk) {
accumulate(&mut acc, x, y);
}
}
acc
}
}
}
pub(crate) fn mod_dot<T: PrimitiveUnsigned>(xs: &[T], ys: &[T], m: T) -> T {
if xs.is_empty() {
return T::ZERO;
}
let d = ModData::new(m, xs.len());
d.reduce_sum(dot_sum(xs, ys, &d))
}
fn mod_dot_vectors<T: PrimitiveUnsigned>(v: &UnsignedVector<T>, w: &UnsignedVector<T>, m: T) -> T {
assert_ne!(m, T::ZERO, "m cannot be zero");
assert_eq!(
v.elements.len(),
w.elements.len(),
"the vectors must have the same dimension"
);
assert!(
v.mod_is_reduced(&m),
"self must be reduced mod m, but {v} has an element >= {m}"
);
assert!(
w.mod_is_reduced(&m),
"other must be reduced mod m, but {w} has an element >= {m}"
);
mod_dot(&v.elements, &w.elements, m)
}
impl<T: PrimitiveUnsigned> ModDot<Self, T> for UnsignedVector<T> {
type Output = T;
#[inline]
fn mod_dot(self, other: Self, m: T) -> T {
mod_dot_vectors(&self, &other, m)
}
}
impl<T: PrimitiveUnsigned> ModDot<&Self, T> for UnsignedVector<T> {
type Output = T;
#[inline]
fn mod_dot(self, other: &Self, m: T) -> T {
mod_dot_vectors(&self, other, m)
}
}
impl<T: PrimitiveUnsigned> ModDot<UnsignedVector<T>, T> for &UnsignedVector<T> {
type Output = T;
#[inline]
fn mod_dot(self, other: UnsignedVector<T>, m: T) -> T {
mod_dot_vectors(self, &other, m)
}
}
impl<T: PrimitiveUnsigned> ModDot<&UnsignedVector<T>, T> for &UnsignedVector<T> {
type Output = T;
#[inline]
fn mod_dot(self, other: &UnsignedVector<T>, m: T) -> T {
mod_dot_vectors(self, other, m)
}
}