use crate::natural::InnerNatural::{Large, Small};
use crate::natural::Natural;
use crate::natural::arithmetic::add::{
limbs_add_limb_to_out, limbs_add_same_length_to_out,
limbs_add_same_length_with_carry_in_in_place_left, limbs_add_same_length_with_carry_in_to_out,
limbs_slice_add_limb_in_place, limbs_slice_add_same_length_in_place_left,
};
use crate::natural::arithmetic::div::{
limbs_div_divide_and_conquer_approx, limbs_div_schoolbook_approx,
};
use crate::natural::arithmetic::mul::mul_mod::{
limbs_mul_mod_base_pow_n_minus_1, limbs_mul_mod_base_pow_n_minus_1_next_size,
limbs_mul_mod_base_pow_n_minus_1_scratch_len,
};
use crate::natural::arithmetic::mul::{
limbs_mul_greater_to_out, limbs_mul_greater_to_out_scratch_len, limbs_mul_same_length_to_out,
limbs_mul_same_length_to_out_scratch_len, limbs_mul_to_out, limbs_mul_to_out_scratch_len,
};
use crate::natural::arithmetic::shl::{limbs_shl_to_out, limbs_slice_shl_in_place};
use crate::natural::arithmetic::shr::{limbs_shr_to_out, limbs_slice_shr_in_place};
use crate::natural::arithmetic::sub::{
limbs_sub_greater_in_place_left, limbs_sub_limb_in_place, limbs_sub_same_length_in_place_left,
limbs_sub_same_length_in_place_right, limbs_sub_same_length_to_out,
limbs_sub_same_length_with_borrow_in_in_place_left,
limbs_sub_same_length_with_borrow_in_in_place_right,
limbs_sub_same_length_with_borrow_in_to_out,
};
use crate::natural::arithmetic::sub_mul::limbs_sub_mul_limb_same_length_in_place_left;
use crate::natural::comparison::cmp::limbs_cmp_same_length;
use crate::natural::logic::not::limbs_not_to_out;
use crate::platform::{
DC_DIV_QR_THRESHOLD, DC_DIVAPPR_Q_THRESHOLD, DoubleLimb, INV_MULMOD_BNM1_THRESHOLD,
INV_NEWTON_THRESHOLD, Limb, MAYBE_DCP1_DIVAPPR, MU_DIV_QR_SKEW_THRESHOLD, MU_DIV_QR_THRESHOLD,
};
use alloc::vec::Vec;
use core::cmp::{Ordering::*, min};
use core::mem::swap;
use malachite_base::num::arithmetic::traits::{
CeilingDivAssignNegMod, CeilingDivNegMod, DivAssignMod, DivAssignRem, DivMod, DivRem,
WrappingAddAssign, WrappingSub, WrappingSubAssign, XMulYToZZ, XXDivModYToQR,
};
use malachite_base::num::basic::integers::PrimitiveInt;
use malachite_base::num::basic::traits::{One, Zero};
use malachite_base::num::basic::unsigneds::PrimitiveUnsigned;
use malachite_base::num::conversion::traits::{HasHalf, JoinHalves, SplitInHalf};
use malachite_base::num::logic::traits::LeadingZeros;
use malachite_base::slices::{slice_move_left, slice_set_zero};
pub_crate_test! {limbs_invert_limb<
DT: From<T> + HasHalf<Half = T> + PrimitiveUnsigned + JoinHalves<Half = T> + SplitInHalf,
T: PrimitiveUnsigned,
>(
d: T,
) -> T {
(DT::join_halves(!d, T::MAX) / DT::from(d)).lower_half()
}}
#[cold]
#[inline(never)]
const fn div_mod_by_preinversion_second_fixup(q_high: Limb, r: Limb, d: Limb) -> (Limb, Limb) {
(q_high.wrapping_add(1), r - d)
}
pub_crate_test! {div_mod_by_preinversion(
n_high: Limb,
n_low: Limb,
d: Limb,
d_inv: Limb
) -> (Limb, Limb) {
let (mut q_high, q_low) = (DoubleLimb::from(n_high) * DoubleLimb::from(d_inv))
.wrapping_add(DoubleLimb::join_halves(n_high.wrapping_add(1), n_low))
.split_in_half();
let mut r = n_low.wrapping_sub(q_high.wrapping_mul(d));
if r > q_low {
let (r_plus_d, overflow) = r.overflowing_add(d);
if overflow {
q_high.wrapping_sub_assign(1);
r = r_plus_d;
}
} else if r >= d {
return div_mod_by_preinversion_second_fixup(q_high, r, d);
}
(q_high, r)
}}
pub_test! {limbs_div_limb_mod(ns: &[Limb], d: Limb) -> (Vec<Limb>, Limb) {
let mut qs = vec![0; ns.len()];
let r = limbs_div_limb_to_out_mod(&mut qs, ns, d);
(qs, r)
}}
pub_crate_test! {limbs_div_limb_to_out_mod(out: &mut [Limb], ns: &[Limb], d: Limb) -> Limb {
assert_ne!(d, 0);
let len = ns.len();
assert!(len > 1);
let out = &mut out[..len];
let bits = LeadingZeros::leading_zeros(d);
if bits == 0 {
let (r, ns_init) = ns.split_last().unwrap();
let mut r = *r;
let (out_last, out_init) = out.split_last_mut().unwrap();
let adjust = r >= d;
if adjust {
r -= d;
}
*out_last = Limb::from(adjust);
let d_inv = limbs_invert_limb::<DoubleLimb, Limb>(d);
for (out_q, &n) in out_init.iter_mut().zip(ns_init.iter()).rev() {
(*out_q, r) = div_mod_by_preinversion(r, n, d, d_inv);
}
r
} else {
let (ns_last, ns_init) = ns.split_last().unwrap();
let (ns, mut r) = if *ns_last < d {
*out.last_mut().unwrap() = 0;
(ns_init, *ns_last)
} else {
(ns, 0)
};
let d = d << bits;
r <<= bits;
let d_inv = limbs_invert_limb::<DoubleLimb, Limb>(d);
let (previous_n, ns_init) = ns.split_last().unwrap();
let mut previous_n = *previous_n;
let cobits = Limb::WIDTH - bits;
r |= previous_n >> cobits;
let (out_head, out_tail) = out.split_first_mut().unwrap();
for (out_q, &n) in out_tail.iter_mut().zip(ns_init.iter()).rev() {
let n_shifted = (previous_n << bits) | (n >> cobits);
(*out_q, r) = div_mod_by_preinversion(r, n_shifted, d, d_inv);
previous_n = n;
}
let out_r;
(*out_head, out_r) = div_mod_by_preinversion(r, previous_n << bits, d, d_inv);
out_r >> bits
}
}}
pub_crate_test! {limbs_div_limb_in_place_mod(ns: &mut [Limb], d: Limb) -> Limb {
assert_ne!(d, 0);
let len = ns.len();
assert!(len > 1);
let bits = LeadingZeros::leading_zeros(d);
let (ns_last, ns_init) = ns.split_last_mut().unwrap();
if bits == 0 {
let mut r = *ns_last;
let adjust = r >= d;
if adjust {
r -= d;
}
*ns_last = Limb::from(adjust);
let d_inv = limbs_invert_limb::<DoubleLimb, Limb>(d);
for n in ns_init.iter_mut().rev() {
(*n, r) = div_mod_by_preinversion(r, *n, d, d_inv);
}
r
} else {
let (ns, mut r) = if *ns_last < d {
let r = *ns_last;
*ns_last = 0;
(ns_init, r)
} else {
(ns, 0)
};
let d = d << bits;
r <<= bits;
let d_inv = limbs_invert_limb::<DoubleLimb, Limb>(d);
let last_index = ns.len() - 1;
let mut previous_n = ns[last_index];
let cobits = Limb::WIDTH - bits;
r |= previous_n >> cobits;
for i in (0..last_index).rev() {
let n = ns[i];
let shifted_n = (previous_n << bits) | (n >> cobits);
(ns[i + 1], r) = div_mod_by_preinversion(r, shifted_n, d, d_inv);
previous_n = n;
}
let out_r;
(ns[0], out_r) = div_mod_by_preinversion(r, previous_n << bits, d, d_inv);
out_r >> bits
}
}}
pub_test! {limbs_div_mod_extra(
out: &mut [Limb],
fraction_len: usize,
mut ns: &[Limb],
d: Limb,
d_inv: Limb,
shift: u64,
) -> Limb {
assert!(!ns.is_empty());
assert_ne!(d, 0);
let (ns_last, ns_init) = ns.split_last().unwrap();
let ns_last = *ns_last;
let d_norm = d << shift;
let (fraction_out, integer_out) = out.split_at_mut(fraction_len);
let mut integer_out = &mut integer_out[..ns.len()];
let mut r;
if shift == 0 {
r = ns_last;
let q_high = r >= d_norm;
if r >= d_norm {
r -= d_norm;
}
let (integer_out_last, integer_out_init) = integer_out.split_last_mut().unwrap();
*integer_out_last = Limb::from(q_high);
for (q, &n) in integer_out_init.iter_mut().zip(ns_init.iter()).rev() {
(*q, r) = div_mod_by_preinversion(r, n, d_norm, d_inv);
}
} else {
r = 0;
if ns_last < d {
r = ns_last << shift;
let integer_out_last;
(integer_out_last, integer_out) = integer_out.split_last_mut().unwrap();
*integer_out_last = 0;
ns = ns_init;
}
if !ns.is_empty() {
let co_shift = Limb::WIDTH - shift;
let (ns_last, ns_init) = ns.split_last().unwrap();
let mut previous_n = *ns_last;
r |= previous_n >> co_shift;
let (integer_out_head, integer_out_tail) = integer_out.split_first_mut().unwrap();
for (q, &n) in integer_out_tail.iter_mut().zip(ns_init.iter()).rev() {
assert!(r < d_norm);
(*q, r) = div_mod_by_preinversion(
r,
(previous_n << shift) | (n >> co_shift),
d_norm,
d_inv,
);
previous_n = n;
}
(*integer_out_head, r) = div_mod_by_preinversion(r, previous_n << shift, d_norm, d_inv);
}
}
for q in fraction_out.iter_mut().rev() {
(*q, r) = div_mod_by_preinversion(r, 0, d_norm, d_inv);
}
r >> shift
}}
pub_crate_test! {limbs_div_mod_extra_in_place(
ns: &mut [Limb],
fraction_len: usize,
d: Limb,
d_inv: Limb,
shift: u64,
) -> Limb {
assert_ne!(d, 0);
let (fraction_ns, mut integer_ns) = ns.split_at_mut(fraction_len);
let ns_last = *integer_ns.last().unwrap();
let d_norm = d << shift;
let mut r;
if shift == 0 {
r = ns_last;
let q_high = r >= d_norm;
if r >= d_norm {
r -= d_norm;
}
let (integer_ns_last, integer_ns_init) = integer_ns.split_last_mut().unwrap();
*integer_ns_last = Limb::from(q_high);
for q in integer_ns_init.iter_mut().rev() {
(*q, r) = div_mod_by_preinversion(r, *q, d_norm, d_inv);
}
} else {
r = 0;
if ns_last < d {
r = ns_last << shift;
let integer_ns_last;
(integer_ns_last, integer_ns) = integer_ns.split_last_mut().unwrap();
*integer_ns_last = 0;
}
if !integer_ns.is_empty() {
let co_shift = Limb::WIDTH - shift;
let mut previous_n = *integer_ns.last().unwrap();
r |= previous_n >> co_shift;
for i in (1..integer_ns.len()).rev() {
assert!(r < d_norm);
let n = integer_ns[i - 1];
(integer_ns[i], r) = div_mod_by_preinversion(
r,
(previous_n << shift) | (n >> co_shift),
d_norm,
d_inv,
);
previous_n = n;
}
(integer_ns[0], r) = div_mod_by_preinversion(r, previous_n << shift, d_norm, d_inv);
}
}
for q in fraction_ns.iter_mut().rev() {
(*q, r) = div_mod_by_preinversion(r, 0, d_norm, d_inv);
}
r >> shift
}}
pub_crate_test! {limbs_two_limb_inverse_helper(hi: Limb, lo: Limb) -> Limb {
let mut d_inv = limbs_invert_limb::<DoubleLimb, Limb>(hi);
let mut hi_product = hi.wrapping_mul(d_inv);
hi_product.wrapping_add_assign(lo);
if hi_product < lo {
d_inv.wrapping_sub_assign(1);
if hi_product >= hi {
hi_product.wrapping_sub_assign(hi);
d_inv.wrapping_sub_assign(1);
}
hi_product.wrapping_sub_assign(hi);
}
let (lo_product_hi, lo_product_lo) = Limb::x_mul_y_to_zz(lo, d_inv);
hi_product.wrapping_add_assign(lo_product_hi);
if hi_product < lo_product_hi {
d_inv.wrapping_sub_assign(1);
if hi_product > hi || hi_product == hi && lo_product_lo >= lo {
d_inv.wrapping_sub_assign(1);
}
}
d_inv
}}
pub_crate_test! {limbs_div_mod_three_limb_by_two_limb(
n_2: Limb,
n_1: Limb,
n_0: Limb,
d_1: Limb,
d_0: Limb,
d_inv: Limb,
) -> (Limb, DoubleLimb) {
let (mut q, q_lo) = (DoubleLimb::from(n_2) * DoubleLimb::from(d_inv))
.wrapping_add(DoubleLimb::join_halves(n_2, n_1))
.split_in_half();
let d = DoubleLimb::join_halves(d_1, d_0);
let mut r = DoubleLimb::join_halves(n_1.wrapping_sub(d_1.wrapping_mul(q)), n_0)
.wrapping_sub(d)
.wrapping_sub(DoubleLimb::from(d_0) * DoubleLimb::from(q));
q.wrapping_add_assign(1);
if r.upper_half() >= q_lo {
let (r_plus_d, overflow) = r.overflowing_add(d);
if overflow {
q.wrapping_sub_assign(1);
r = r_plus_d;
}
} else if r >= d {
q.wrapping_add_assign(1);
r.wrapping_sub_assign(d);
}
(q, r)
}}
pub_crate_test! {limbs_div_mod_by_two_limb_normalized(
qs: &mut [Limb],
ns: &mut [Limb],
ds: &[Limb]
) -> bool {
assert_eq!(ds.len(), 2);
let n_len = ns.len();
assert!(n_len >= 2);
let n_limit = n_len - 2;
assert!(ds[1].get_highest_bit());
let d_1 = ds[1];
let d_0 = ds[0];
let d = DoubleLimb::join_halves(d_1, d_0);
let mut r = DoubleLimb::join_halves(ns[n_limit + 1], ns[n_limit]);
let highest_q = r >= d;
if highest_q {
r.wrapping_sub_assign(d);
}
let (mut r_1, mut r_0) = r.split_in_half();
let d_inv = limbs_two_limb_inverse_helper(d_1, d_0);
for (&n, q) in ns[..n_limit].iter().zip(qs[..n_limit].iter_mut()).rev() {
let r;
(*q, r) = limbs_div_mod_three_limb_by_two_limb(r_1, r_0, n, d_1, d_0, d_inv);
(r_1, r_0) = r.split_in_half();
}
ns[1] = r_1;
ns[0] = r_0;
highest_q
}}
pub_crate_test! {limbs_div_mod_schoolbook(
qs: &mut [Limb],
ns: &mut [Limb],
ds: &[Limb],
d_inv: Limb,
) -> bool {
let d_len = ds.len();
assert!(d_len > 2);
let n_len = ns.len();
assert!(n_len >= d_len);
let d_1 = ds[d_len - 1];
assert!(d_1.get_highest_bit());
let d_0 = ds[d_len - 2];
let ds_except_last = &ds[..d_len - 1];
let ds_except_last_two = &ds[..d_len - 2];
let ns_hi = &mut ns[n_len - d_len..];
let highest_q = limbs_cmp_same_length(ns_hi, ds) >= Equal;
if highest_q {
limbs_sub_same_length_in_place_left(ns_hi, ds);
}
let mut n_1 = ns[n_len - 1];
for i in (d_len..n_len).rev() {
let j = i - d_len;
let mut q;
if n_1 == d_1 && ns[i - 1] == d_0 {
q = Limb::MAX;
limbs_sub_mul_limb_same_length_in_place_left(&mut ns[j..i], ds, q);
n_1 = ns[i - 1]; } else {
let (ns_lo, ns_hi) = ns.split_at_mut(i - 2);
let n;
(q, n) = limbs_div_mod_three_limb_by_two_limb(n_1, ns_hi[1], ns_hi[0], d_1, d_0, d_inv);
let mut n_0;
(n_1, n_0) = n.split_in_half();
let local_carry_1 = limbs_sub_mul_limb_same_length_in_place_left(
&mut ns_lo[j..],
ds_except_last_two,
q,
);
let local_carry_2 = n_0 < local_carry_1;
n_0.wrapping_sub_assign(local_carry_1);
let carry = local_carry_2 && n_1 == 0;
if local_carry_2 {
n_1.wrapping_sub_assign(1);
}
ns_hi[0] = n_0;
if carry {
n_1.wrapping_add_assign(d_1);
if limbs_slice_add_same_length_in_place_left(&mut ns[j..i - 1], ds_except_last) {
n_1.wrapping_add_assign(1);
}
q.wrapping_sub_assign(1);
}
}
qs[j] = q;
}
ns[d_len - 1] = n_1;
highest_q
}}
pub(crate) fn limbs_div_mod_divide_and_conquer_helper(
qs: &mut [Limb],
ns: &mut [Limb],
ds: &[Limb],
d_inv: Limb,
scratch: &mut [Limb],
) -> bool {
let n = ds.len();
let lo = n >> 1; let hi = n - lo; let qs_hi = &mut qs[lo..];
let (ds_lo, ds_hi) = ds.split_at(lo);
let mut highest_q = if hi < DC_DIV_QR_THRESHOLD {
limbs_div_mod_schoolbook(qs_hi, &mut ns[lo << 1..n << 1], ds_hi, d_inv)
} else {
limbs_div_mod_divide_and_conquer_helper(qs_hi, &mut ns[lo << 1..], ds_hi, d_inv, scratch)
};
let qs_hi = &mut qs_hi[..hi];
let mut mul_scratch = vec![0; limbs_mul_greater_to_out_scratch_len(qs_hi.len(), ds_lo.len())];
limbs_mul_greater_to_out(scratch, qs_hi, ds_lo, &mut mul_scratch);
let ns_lo = &mut ns[..n + lo];
let mut carry = Limb::from(limbs_sub_same_length_in_place_left(
&mut ns_lo[lo..],
&scratch[..n],
));
if highest_q && limbs_sub_same_length_in_place_left(&mut ns_lo[n..], ds_lo) {
carry += 1;
}
while carry != 0 {
if limbs_sub_limb_in_place(qs_hi, 1) {
assert!(highest_q);
highest_q = false;
}
if limbs_slice_add_same_length_in_place_left(&mut ns_lo[lo..], ds) {
carry -= 1;
}
}
let (ds_lo, ds_hi) = ds.split_at(hi);
let q_lo = if lo < DC_DIV_QR_THRESHOLD {
limbs_div_mod_schoolbook(qs, &mut ns[hi..n + lo], ds_hi, d_inv)
} else {
limbs_div_mod_divide_and_conquer_helper(qs, &mut ns[hi..], ds_hi, d_inv, scratch)
};
let qs_lo = &mut qs[..lo];
let ns_lo = &mut ns[..n];
let mut mul_scratch = vec![0; limbs_mul_greater_to_out_scratch_len(ds_lo.len(), lo)];
limbs_mul_greater_to_out(scratch, ds_lo, qs_lo, &mut mul_scratch);
let mut carry = Limb::from(limbs_sub_same_length_in_place_left(ns_lo, &scratch[..n]));
if q_lo && limbs_sub_same_length_in_place_left(&mut ns_lo[lo..], ds_lo) {
carry += 1;
}
while carry != 0 {
limbs_sub_limb_in_place(qs_lo, 1);
if limbs_slice_add_same_length_in_place_left(ns_lo, ds) {
carry -= 1;
}
}
highest_q
}
pub_test! {limbs_div_dc_helper(
qs: &mut [Limb],
ns: &mut [Limb],
ds: &[Limb],
d_inv: Limb,
scratch: &mut [Limb],
) -> bool {
if qs.len() < DC_DIV_QR_THRESHOLD {
limbs_div_mod_schoolbook(qs, ns, ds, d_inv)
} else {
limbs_div_mod_divide_and_conquer_helper(qs, ns, ds, d_inv, scratch)
}
}}
pub_test! {limbs_div_mod_divide_and_conquer(
qs: &mut [Limb],
ns: &mut [Limb],
ds: &[Limb],
d_inv: Limb,
) -> bool {
let n_len = ns.len();
let d_len = ds.len();
assert!(d_len >= 6); assert!(n_len >= d_len + 3); let a = d_len - 1;
let d_1 = ds[a];
let b = d_len - 2;
assert!(d_1.get_highest_bit());
let mut scratch = vec![0; d_len];
let mut highest_q;
let q_len = n_len - d_len;
if q_len > d_len {
let q_len_mod_d_len = {
let mut m = q_len % d_len;
if m == 0 {
m = d_len;
}
m
};
let qs_alt = &mut qs[q_len - q_len_mod_d_len..q_len];
if q_len_mod_d_len == 1 {
let ns_hi = &mut ns[q_len - 1..];
let ns_hi_hi = &mut ns_hi[1..];
highest_q = limbs_cmp_same_length(ns_hi_hi, ds) >= Equal;
if highest_q {
assert!(!limbs_sub_same_length_in_place_left(ns_hi_hi, ds));
}
let (last_n, ns) = ns_hi.split_last_mut().unwrap();
let n_2 = *last_n;
let mut n_1 = ns[a];
let mut n_0 = ns[b];
let d_0 = ds[b];
assert!(n_2 < d_1 || n_2 == d_1 && n_1 <= d_0);
let mut q;
if n_2 == d_1 && n_1 == d_0 {
q = Limb::MAX;
assert_eq!(limbs_sub_mul_limb_same_length_in_place_left(ns, ds, q), n_2);
} else {
let n;
(q, n) = limbs_div_mod_three_limb_by_two_limb(n_2, n_1, n_0, d_1, d_0, d_inv);
(n_1, n_0) = n.split_in_half();
let local_carry_1 =
limbs_sub_mul_limb_same_length_in_place_left(&mut ns[..b], &ds[..b], q);
let local_carry_2 = n_0 < local_carry_1;
n_0.wrapping_sub_assign(local_carry_1);
let carry = local_carry_2 && n_1 == 0;
if local_carry_2 {
n_1.wrapping_sub_assign(1);
}
ns[b] = n_0;
let (last_n, ns) = ns.split_last_mut().unwrap();
if carry {
n_1.wrapping_add_assign(d_1);
if limbs_slice_add_same_length_in_place_left(ns, &ds[..a]) {
n_1.wrapping_add_assign(1);
}
if q == 0 {
assert!(highest_q);
highest_q = false;
}
q.wrapping_sub_assign(1);
}
*last_n = n_1;
}
qs_alt[0] = q;
} else {
let (ds_lo, ds_hi) = ds.split_at(d_len - q_len_mod_d_len);
highest_q = {
let ns = &mut ns[n_len - (q_len_mod_d_len << 1)..];
if q_len_mod_d_len == 2 {
limbs_div_mod_by_two_limb_normalized(qs_alt, ns, ds_hi)
} else {
limbs_div_dc_helper(qs_alt, ns, ds_hi, d_inv, &mut scratch)
}
};
if q_len_mod_d_len != d_len {
let mut mul_scratch =
vec![0; limbs_mul_to_out_scratch_len(qs_alt.len(), ds_lo.len())];
limbs_mul_to_out(&mut scratch, qs_alt, ds_lo, &mut mul_scratch);
let ns = &mut ns[q_len - q_len_mod_d_len..n_len - q_len_mod_d_len];
let mut carry = Limb::from(limbs_sub_same_length_in_place_left(ns, &scratch));
if highest_q
&& limbs_sub_same_length_in_place_left(&mut ns[q_len_mod_d_len..], ds_lo)
{
carry += 1;
}
while carry != 0 {
if limbs_sub_limb_in_place(qs_alt, 1) {
assert!(highest_q);
highest_q = false;
}
if limbs_slice_add_same_length_in_place_left(ns, ds) {
carry -= 1;
}
}
}
}
let mut offset = n_len.checked_sub(d_len + q_len_mod_d_len).unwrap();
while offset != 0 {
offset -= d_len;
limbs_div_mod_divide_and_conquer_helper(
&mut qs[offset..],
&mut ns[offset..],
ds,
d_inv,
&mut scratch,
);
}
} else {
let m = d_len - q_len;
let (ds_lo, ds_hi) = ds.split_at(m);
highest_q = if q_len < DC_DIV_QR_THRESHOLD {
limbs_div_mod_schoolbook(qs, &mut ns[m..], ds_hi, d_inv)
} else {
limbs_div_mod_divide_and_conquer_helper(qs, &mut ns[m..], ds_hi, d_inv, &mut scratch)
};
if m != 0 {
let qs = &mut qs[..q_len];
let ns = &mut ns[..d_len];
let mut mul_scratch = vec![0; limbs_mul_to_out_scratch_len(q_len, ds_lo.len())];
limbs_mul_to_out(&mut scratch, qs, ds_lo, &mut mul_scratch);
let mut carry = Limb::from(limbs_sub_same_length_in_place_left(ns, &scratch));
if highest_q && limbs_sub_same_length_in_place_left(&mut ns[q_len..], ds_lo) {
carry += 1;
}
while carry != 0 {
if limbs_sub_limb_in_place(qs, 1) {
assert!(highest_q);
highest_q = false;
}
if limbs_slice_add_same_length_in_place_left(ns, ds) {
carry -= 1;
}
}
}
}
highest_q
}}
pub_test! {limbs_div_approx_helper(qs: &mut [Limb], ns: &mut [Limb], ds: &[Limb], d_inv: Limb) {
if ds.len() < DC_DIVAPPR_Q_THRESHOLD {
limbs_div_schoolbook_approx(qs, ns, ds, d_inv);
} else {
limbs_div_divide_and_conquer_approx(qs, ns, ds, d_inv);
}
}}
pub_test! {limbs_invert_basecase_approx(
is: &mut [Limb],
ds: &[Limb],
scratch: &mut [Limb]
) -> bool {
let d_len = ds.len();
assert_ne!(d_len, 0);
let highest_d = ds[d_len - 1];
assert!(highest_d.get_highest_bit());
if d_len == 1 {
let d = ds[0];
is[0] = limbs_invert_limb::<DoubleLimb, Limb>(d);
} else {
let scratch = &mut scratch[..d_len << 1];
let (scratch_lo, scratch_hi) = scratch.split_at_mut(d_len);
for s in &mut *scratch_lo {
*s = Limb::MAX;
}
limbs_not_to_out(scratch_hi, ds);
if d_len == 2 {
limbs_div_mod_by_two_limb_normalized(is, scratch, ds);
} else {
let d_inv = limbs_two_limb_inverse_helper(highest_d, ds[d_len - 2]);
if MAYBE_DCP1_DIVAPPR {
limbs_div_approx_helper(is, scratch, ds, d_inv);
} else {
limbs_div_schoolbook_approx(is, scratch, ds, d_inv);
}
assert!(!limbs_sub_limb_in_place(&mut is[..d_len], 1));
return false;
}
}
true
}}
pub_test! {limbs_invert_newton_approx(is: &mut [Limb], ds: &[Limb], scratch: &mut [Limb]) -> bool {
let d_len = ds.len();
assert!(d_len > 4);
assert!(ds[d_len - 1].get_highest_bit());
let is = &mut is[..d_len];
let mut size = d_len;
let mut sizes = vec![size];
size = (size >> 1) + 1;
let mut scratch2 = vec![];
let mut mul_size = 0;
if d_len >= INV_MULMOD_BNM1_THRESHOLD {
mul_size = limbs_mul_mod_base_pow_n_minus_1_next_size(d_len + 1);
scratch2 = vec![0; limbs_mul_mod_base_pow_n_minus_1_scratch_len(mul_size, d_len, size)];
}
while size >= INV_NEWTON_THRESHOLD {
sizes.push(size);
size = (size >> 1) + 1;
}
limbs_invert_basecase_approx(&mut is[d_len - size..], &ds[d_len - size..], scratch);
let mut previous_size = size;
let mut a = 0;
for &size in sizes.iter().rev() {
let ds_hi = &ds[d_len - size..];
let condition = size < INV_MULMOD_BNM1_THRESHOLD || {
mul_size = limbs_mul_mod_base_pow_n_minus_1_next_size(size + 1);
mul_size > size + previous_size
};
let diff = size - previous_size;
let is_hi = &mut is[d_len - previous_size..];
if condition {
let mut mul_scratch =
vec![0; limbs_mul_greater_to_out_scratch_len(ds_hi.len(), is_hi.len())];
limbs_mul_greater_to_out(scratch, ds_hi, is_hi, &mut mul_scratch);
limbs_slice_add_same_length_in_place_left(
&mut scratch[previous_size..=size],
&ds_hi[..=diff],
);
} else {
limbs_mul_mod_base_pow_n_minus_1(scratch, mul_size, ds_hi, is_hi, &mut scratch2);
let scratch = &mut scratch[..=mul_size];
let mul_diff = mul_size - previous_size;
assert!(size >= mul_diff);
let (ds_hi_lo, ds_hi_hi) = ds_hi.split_at(mul_diff);
let carry = limbs_slice_add_same_length_in_place_left(
&mut scratch[previous_size..mul_size],
ds_hi_lo,
);
scratch[mul_size] = 1; let (scratch_lo, scratch_hi) = scratch.split_at_mut(size - mul_diff);
if !limbs_add_same_length_with_carry_in_in_place_left(scratch_lo, ds_hi_hi, carry) {
assert!(!limbs_sub_limb_in_place(scratch_hi, 1));
}
let (scratch_last, scratch_init) = scratch.split_last_mut().unwrap();
assert!(!limbs_sub_limb_in_place(
scratch_init,
1.wrapping_sub(*scratch_last)
));
}
if scratch[size] < 2 {
let (scratch_lo, scratch_hi) = scratch.split_at_mut(size);
let mut carry = scratch_hi[0] + 1; if carry == 2 && !limbs_sub_same_length_in_place_left(scratch_lo, ds_hi) {
carry = 3;
assert!(limbs_sub_same_length_in_place_left(scratch_lo, ds_hi));
}
if limbs_cmp_same_length(scratch_lo, ds_hi) == Greater {
assert!(!limbs_sub_same_length_in_place_left(scratch_lo, ds_hi));
carry += 1;
}
let (scratch_lo, scratch_mid) = scratch_lo.split_at_mut(diff);
let (ds_hi_lo, ds_hi_hi) = ds_hi.split_at(diff);
let borrow = limbs_cmp_same_length(scratch_lo, ds_hi_lo) == Greater;
assert!(!limbs_sub_same_length_with_borrow_in_to_out(
&mut scratch_hi[diff..],
ds_hi_hi,
scratch_mid,
borrow
));
assert!(!limbs_sub_limb_in_place(is_hi, carry)); } else {
assert!(scratch[size] >= Limb::MAX - 1);
if condition {
assert!(!limbs_sub_limb_in_place(&mut scratch[..=size], 1));
}
let (scratch_lo, scratch_hi) = scratch.split_at_mut(size);
if scratch_hi[0] != Limb::MAX {
assert!(!limbs_slice_add_limb_in_place(is_hi, 1));
assert!(limbs_slice_add_same_length_in_place_left(scratch_lo, ds_hi));
}
limbs_not_to_out(&mut scratch_hi[diff..size], &scratch_lo[diff..]);
}
let (scratch_lo, scratch_hi) = scratch.split_at_mut(size + diff);
let mut mul_scratch = vec![0; limbs_mul_same_length_to_out_scratch_len(is_hi.len())];
limbs_mul_same_length_to_out(
scratch_lo,
&scratch_hi[..previous_size],
is_hi,
&mut mul_scratch,
);
a = (previous_size << 1) - diff;
let carry = {
let (scratch_lo, scratch_hi) = scratch.split_at_mut(a);
limbs_slice_add_same_length_in_place_left(
&mut scratch_lo[previous_size..],
&scratch_hi[3 * diff - previous_size..diff << 1],
)
};
if limbs_add_same_length_with_carry_in_to_out(
&mut is[d_len - size..],
&scratch[a..previous_size << 1],
&scratch[size + previous_size..size << 1],
carry,
) {
assert!(!limbs_slice_add_limb_in_place(
&mut is[d_len - previous_size..],
1
));
}
previous_size = size;
}
scratch[a - 1] <= Limb::MAX - 7
}}
pub_crate_test! {limbs_invert_approx(is: &mut [Limb], ds: &[Limb], scratch: &mut [Limb]) -> bool {
if ds.len() < INV_NEWTON_THRESHOLD {
limbs_invert_basecase_approx(is, ds, scratch)
} else {
limbs_invert_newton_approx(is, ds, scratch)
}
}}
pub(crate) const MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD: usize = INV_MULMOD_BNM1_THRESHOLD >> 1;
pub_crate_test! {limbs_div_barrett_large_product(
scratch: &mut [Limb],
ds: &[Limb],
qs: &[Limb],
rs_hi: &[Limb],
scratch_len: usize,
i_len: usize,
) {
let d_len = ds.len();
let (scratch, scratch_out) = scratch.split_at_mut(scratch_len);
limbs_mul_mod_base_pow_n_minus_1(scratch, scratch_len, ds, qs, scratch_out);
if d_len + i_len > scratch_len {
let (rs_hi_lo, rs_hi_hi) = rs_hi.split_at(scratch_len - d_len);
let carry_1 = limbs_sub_greater_in_place_left(scratch, rs_hi_hi);
let carry_2 = limbs_cmp_same_length(rs_hi_lo, &scratch[d_len..]) == Less;
if !carry_1 && carry_2 {
assert!(!limbs_slice_add_limb_in_place(scratch, 1));
} else {
assert_eq!(carry_1, carry_2);
}
}
}}
fn limbs_div_mod_barrett_preinverted(
qs: &mut [Limb],
rs: &mut [Limb],
ns: &[Limb],
ds: &[Limb],
mut is: &[Limb],
scratch: &mut [Limb],
) -> bool {
let n_len = ns.len();
let d_len = ds.len();
assert_eq!(rs.len(), d_len);
let mut i_len = is.len();
let q_len = n_len - d_len;
let qs = &mut qs[..q_len];
let (ns_lo, ns_hi) = ns.split_at(q_len);
let highest_q = limbs_cmp_same_length(ns_hi, ds) >= Equal;
if highest_q {
limbs_sub_same_length_to_out(rs, ns_hi, ds);
} else {
rs.copy_from_slice(ns_hi);
}
let scratch_len = if i_len < MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD {
0
} else {
limbs_mul_mod_base_pow_n_minus_1_next_size(d_len + 1)
};
let mut n = d_len - i_len;
for (ns, qs) in ns_lo.rchunks(i_len).zip(qs.rchunks_mut(i_len)) {
let chunk_len = ns.len();
if i_len != chunk_len {
is = &is[i_len - chunk_len..];
i_len = chunk_len;
n = d_len - i_len;
}
let (rs_lo, rs_hi) = rs.split_at_mut(n);
let mut mul_scratch = vec![0; limbs_mul_same_length_to_out_scratch_len(is.len())];
limbs_mul_same_length_to_out(scratch, rs_hi, is, &mut mul_scratch);
assert!(!limbs_add_same_length_to_out(
qs,
&scratch[i_len..i_len << 1],
rs_hi,
));
if i_len < MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD {
let mut mul_scratch = vec![0; limbs_mul_greater_to_out_scratch_len(ds.len(), qs.len())];
limbs_mul_greater_to_out(scratch, ds, qs, &mut mul_scratch);
} else {
limbs_div_barrett_large_product(scratch, ds, qs, rs_hi, scratch_len, i_len);
}
let mut r = rs_hi[0].wrapping_sub(scratch[d_len]);
let scratch = &mut scratch[..d_len];
let carry = if n == 0 {
limbs_sub_same_length_to_out(rs, ns, scratch)
} else {
let (scratch_lo, scratch_hi) = scratch.split_at_mut(i_len);
let carry = limbs_sub_same_length_with_borrow_in_in_place_right(
rs_lo,
scratch_hi,
limbs_sub_same_length_in_place_right(ns, scratch_lo),
);
rs.copy_from_slice(scratch);
carry
};
if carry {
r.wrapping_sub_assign(1);
}
while r != 0 {
assert!(!limbs_slice_add_limb_in_place(qs, 1));
if limbs_sub_same_length_in_place_left(rs, ds) {
r -= 1;
}
}
if limbs_cmp_same_length(rs, ds) >= Equal {
assert!(!limbs_slice_add_limb_in_place(qs, 1));
limbs_sub_same_length_in_place_left(rs, ds);
}
}
highest_q
}
pub_const_crate_test! {limbs_div_mod_barrett_is_len(q_len: usize, d_len: usize) -> usize {
let q_len_minus_1 = q_len - 1;
if q_len > d_len {
let b = q_len_minus_1 / d_len + 1; q_len_minus_1 / b + 1 } else if 3 * q_len > d_len {
(q_len_minus_1 >> 1) + 1 } else {
q_len_minus_1 + 1 }
}}
pub_crate_test! {limbs_div_mod_barrett_helper(
qs: &mut [Limb],
rs: &mut [Limb],
ns: &[Limb],
ds: &[Limb],
scratch: &mut [Limb],
) -> bool {
let n_len = ns.len();
let d_len = ds.len();
assert_eq!(rs.len(), d_len);
assert!(d_len > 1);
assert!(n_len > d_len);
let q_len = n_len - d_len;
let i_len = limbs_div_mod_barrett_is_len(q_len, d_len);
assert!(i_len <= d_len);
let i_len_plus_1 = i_len + 1;
let (is, scratch_hi) = scratch.split_at_mut(i_len_plus_1);
if d_len == i_len {
let (scratch_lo, scratch_hi) = scratch_hi.split_at_mut(i_len_plus_1);
let (scratch_first, scratch_lo_tail) = scratch_lo.split_first_mut().unwrap();
scratch_lo_tail.copy_from_slice(&ds[..i_len]);
*scratch_first = 1;
limbs_invert_approx(is, scratch_lo, scratch_hi);
slice_move_left(is, 1);
} else if limbs_add_limb_to_out(scratch_hi, &ds[d_len - i_len_plus_1..], 1) {
slice_set_zero(&mut is[..i_len]);
} else {
let (scratch_lo, scratch_hi) = scratch_hi.split_at_mut(i_len_plus_1);
limbs_invert_approx(is, scratch_lo, scratch_hi);
slice_move_left(is, 1);
}
let (scratch_lo, scratch_hi) = scratch.split_at_mut(i_len);
limbs_div_mod_barrett_preinverted(qs, rs, ns, ds, scratch_lo, scratch_hi)
}}
fn limbs_div_mod_barrett_preinverse_scratch_len(d_len: usize, is_len: usize) -> usize {
let itch_local = limbs_mul_mod_base_pow_n_minus_1_next_size(d_len + 1);
let itch_out = limbs_mul_mod_base_pow_n_minus_1_scratch_len(itch_local, d_len, is_len);
itch_local + itch_out
}
pub(crate) const fn limbs_invert_approx_scratch_len(is_len: usize) -> usize {
is_len << 1
}
pub_crate_test! {limbs_div_mod_barrett_scratch_len(n_len: usize, d_len: usize) -> usize {
let is_len = limbs_div_mod_barrett_is_len(n_len - d_len, d_len);
let preinverse_len = limbs_div_mod_barrett_preinverse_scratch_len(d_len, is_len);
let inv_approx_len = limbs_invert_approx_scratch_len(is_len + 1) + is_len + 2;
assert!(preinverse_len >= inv_approx_len);
is_len + preinverse_len
}}
pub_test! {limbs_div_mod_barrett_large_helper(
qs: &mut [Limb],
rs: &mut [Limb],
ns: &[Limb],
ds: &[Limb],
scratch: &mut [Limb],
) -> bool {
let n_len = ns.len();
let d_len = ds.len();
let q_len = qs.len();
let q_len_plus_one = q_len + 1;
let n = n_len - q_len - q_len_plus_one; let (ns_lo, ns_hi) = ns.split_at(n);
let (ds_lo, ds_hi) = ds.split_at(d_len - q_len_plus_one);
let (rs_lo, rs_hi) = rs.split_at_mut(n);
let rs_hi = &mut rs_hi[..q_len_plus_one];
let mut highest_q = limbs_div_mod_barrett_helper(qs, rs_hi, ns_hi, ds_hi, scratch);
let mut mul_scratch = vec![0; limbs_mul_to_out_scratch_len(ds_lo.len(), qs.len())];
limbs_mul_to_out(scratch, ds_lo, qs, &mut mul_scratch);
let (scratch_last, scratch_init) = scratch[..d_len].split_last_mut().unwrap();
*scratch_last = Limb::from(
highest_q && limbs_slice_add_same_length_in_place_left(&mut scratch_init[q_len..], ds_lo),
);
let (scratch_lo, scratch_hi) = scratch.split_at(n);
let scratch_hi = &scratch_hi[..q_len_plus_one];
if limbs_sub_same_length_with_borrow_in_in_place_left(
rs_hi,
scratch_hi,
limbs_sub_same_length_to_out(rs_lo, ns_lo, scratch_lo),
) {
if limbs_sub_limb_in_place(qs, 1) {
assert!(highest_q);
highest_q = false;
}
limbs_slice_add_same_length_in_place_left(&mut rs[..d_len], ds);
}
highest_q
}}
pub_test! {limbs_div_mod_barrett(
qs: &mut [Limb],
rs: &mut [Limb],
ns: &[Limb],
ds: &[Limb],
scratch: &mut [Limb],
) -> bool {
let n_len = ns.len();
let d_len = ds.len();
let q_len = n_len - d_len;
let qs = &mut qs[..q_len];
if d_len <= q_len + MU_DIV_QR_SKEW_THRESHOLD {
limbs_div_mod_barrett_helper(qs, &mut rs[..d_len], ns, ds, scratch)
} else {
limbs_div_mod_barrett_large_helper(qs, rs, ns, ds, scratch)
}
}}
fn limbs_div_mod_by_two_limb(qs: &mut [Limb], rs: &mut [Limb], ns: &[Limb], ds: &[Limb]) {
let n_len = ns.len();
let ds_1 = ds[1];
let bits = LeadingZeros::leading_zeros(ds_1);
if bits == 0 {
let mut ns = ns.to_vec();
qs[n_len - 2] = Limb::from(limbs_div_mod_by_two_limb_normalized(qs, &mut ns, ds));
rs[0] = ns[0];
rs[1] = ns[1];
} else {
let ds_0 = ds[0];
let cobits = Limb::WIDTH - bits;
let mut ns_shifted = vec![0; n_len + 1];
let ns_shifted = &mut ns_shifted;
let carry = limbs_shl_to_out(ns_shifted, ns, bits);
let ds_shifted = &mut [ds_0 << bits, (ds_1 << bits) | (ds_0 >> cobits)];
if carry == 0 {
qs[n_len - 2] = Limb::from(limbs_div_mod_by_two_limb_normalized(
qs,
&mut ns_shifted[..n_len],
ds_shifted,
));
} else {
ns_shifted[n_len] = carry;
limbs_div_mod_by_two_limb_normalized(qs, ns_shifted, ds_shifted);
}
let ns_shifted_1 = ns_shifted[1];
rs[0] = (ns_shifted[0] >> bits) | (ns_shifted_1 << cobits);
rs[1] = ns_shifted_1 >> bits;
}
}
pub(crate) const MUPI_DIV_QR_THRESHOLD: usize = 74;
fn limbs_div_mod_dc_condition(n_len: usize, d_len: usize) -> bool {
let n_64 = n_len as f64;
let d_64 = d_len as f64;
d_len < MUPI_DIV_QR_THRESHOLD
|| n_len < MU_DIV_QR_THRESHOLD << 1
|| fma!(
((MU_DIV_QR_THRESHOLD - MUPI_DIV_QR_THRESHOLD) << 1) as f64,
d_64,
MUPI_DIV_QR_THRESHOLD as f64 * n_64
) > d_64 * n_64
}
fn limbs_div_mod_unbalanced(
qs: &mut [Limb],
rs: &mut [Limb],
ns: &[Limb],
ds: &[Limb],
adjusted_n_len: usize,
) {
let mut n_len = ns.len();
let d_len = ds.len();
qs[n_len - d_len] = 0; let mut ds_shifted_vec;
let ds_shifted: &[Limb];
let mut ns_shifted_vec = vec![0; n_len + 1];
let ns_shifted = &mut ns_shifted_vec;
let bits = LeadingZeros::leading_zeros(*ds.last().unwrap());
if bits == 0 {
ds_shifted = ds;
ns_shifted[..n_len].copy_from_slice(ns);
} else {
ds_shifted_vec = vec![0; d_len];
limbs_shl_to_out(&mut ds_shifted_vec, ds, bits);
ds_shifted = &ds_shifted_vec;
let (ns_shifted_last, ns_shifted_init) = ns_shifted.split_last_mut().unwrap();
*ns_shifted_last = limbs_shl_to_out(ns_shifted_init, ns, bits);
}
n_len = adjusted_n_len;
let d_inv = limbs_two_limb_inverse_helper(ds_shifted[d_len - 1], ds_shifted[d_len - 2]);
let ns_shifted = &mut ns_shifted[..n_len];
if d_len < DC_DIV_QR_THRESHOLD {
limbs_div_mod_schoolbook(qs, ns_shifted, ds_shifted, d_inv);
let ns_shifted = &ns_shifted[..d_len];
if bits == 0 {
rs.copy_from_slice(ns_shifted);
} else {
limbs_shr_to_out(rs, ns_shifted, bits);
}
} else if limbs_div_mod_dc_condition(n_len, d_len) {
limbs_div_mod_divide_and_conquer(qs, ns_shifted, ds_shifted, d_inv);
let ns_shifted = &ns_shifted[..d_len];
if bits == 0 {
rs.copy_from_slice(ns_shifted);
} else {
limbs_shr_to_out(rs, ns_shifted, bits);
}
} else {
let scratch_len = limbs_div_mod_barrett_scratch_len(n_len, d_len);
let mut scratch = vec![0; scratch_len];
limbs_div_mod_barrett(qs, rs, ns_shifted, ds_shifted, &mut scratch);
if bits != 0 {
limbs_slice_shr_in_place(rs, bits);
}
}
}
pub(crate) fn limbs_div_mod_balanced(
qs: &mut [Limb],
rs: &mut [Limb],
ns: &[Limb],
ds: &[Limb],
adjust: bool,
) {
let n_len = ns.len();
let d_len = ds.len();
let mut q_len = n_len - d_len;
assert!(d_len >= q_len);
qs[q_len] = 0; if adjust {
q_len += 1;
} else if q_len == 0 {
rs.copy_from_slice(&ns[..d_len]);
return;
}
let q_len = q_len;
let i_len = d_len - q_len;
let bits = LeadingZeros::leading_zeros(ds[d_len - 1]);
let cobits = Limb::WIDTH - bits;
let q_len_2 = q_len << 1;
let m = n_len - q_len_2;
let mut ns_shifted_vec = vec![0; q_len_2 + 1];
let mut ds_shifted_vec;
let ds_shifted: &[Limb];
let ds_hi = &ds[i_len..];
let ds_lo_last = ds[i_len - 1];
let carry = if bits == 0 {
ds_shifted = ds_hi;
ns_shifted_vec[..q_len_2].copy_from_slice(&ns[m..]);
0
} else {
ds_shifted_vec = vec![0; q_len];
limbs_shl_to_out(&mut ds_shifted_vec, ds_hi, bits);
ds_shifted_vec[0] |= ds_lo_last >> cobits;
ds_shifted = &ds_shifted_vec;
let carry = limbs_shl_to_out(&mut ns_shifted_vec, &ns[m..], bits);
if !adjust {
ns_shifted_vec[0] |= ns[m - 1] >> cobits;
}
carry
};
let ns_shifted = if adjust {
ns_shifted_vec[q_len_2] = carry;
&mut ns_shifted_vec[1..]
} else {
&mut ns_shifted_vec
};
if q_len == 1 {
(qs[0], ns_shifted[0]) =
Limb::xx_div_mod_y_to_qr(ns_shifted[1], ns_shifted[0], ds_shifted[0]);
} else if q_len == 2 {
limbs_div_mod_by_two_limb_normalized(qs, ns_shifted, ds_shifted);
} else {
let ns_shifted = &mut ns_shifted[..q_len_2];
let d_inv = limbs_two_limb_inverse_helper(ds_shifted[q_len - 1], ds_shifted[q_len - 2]);
if q_len < DC_DIV_QR_THRESHOLD {
limbs_div_mod_schoolbook(qs, ns_shifted, ds_shifted, d_inv);
} else if q_len < MU_DIV_QR_THRESHOLD {
limbs_div_mod_divide_and_conquer(qs, ns_shifted, ds_shifted, d_inv);
} else {
let mut scratch = vec![0; limbs_div_mod_barrett_scratch_len(q_len_2, q_len)];
limbs_div_mod_barrett(qs, rs, ns_shifted, ds_shifted, &mut scratch);
ns_shifted[..q_len].copy_from_slice(&rs[..q_len]);
}
}
let mut r_len = q_len;
let mut x = ds_lo_last << bits;
if i_len >= 2 {
x |= ds[i_len - 2] >> 1 >> (!bits & Limb::WIDTH_MASK);
}
if ns_shifted[q_len - 1] < (DoubleLimb::from(x) * DoubleLimb::from(qs[q_len - 1])).upper_half()
{
assert!(!limbs_sub_limb_in_place(qs, 1));
let carry = limbs_slice_add_same_length_in_place_left(&mut ns_shifted[..q_len], ds_shifted);
if carry {
ns_shifted[q_len] = 1;
r_len += 1;
}
}
let mut q_too_large = false;
let mut do_extra_cleanup = true;
let mut scratch = vec![0; d_len];
let mut i_len_alt = i_len;
let qs_lo = &mut qs[..q_len];
if bits != 0 {
let carry_1 = limbs_slice_shl_in_place(&mut ns_shifted[..r_len], cobits);
let mask = Limb::MAX >> bits;
ns_shifted[0] |= ns[i_len - 1] & mask;
let (ns_shifted_last, ns_shifted_init) = ns_shifted[..=q_len].split_last_mut().unwrap();
let carry_2 = limbs_sub_mul_limb_same_length_in_place_left(
ns_shifted_init,
qs_lo,
ds[i_len - 1] & mask,
);
if q_len == r_len {
(*ns_shifted_last, q_too_large) = carry_1.overflowing_sub(carry_2);
r_len += 1;
} else {
assert!(*ns_shifted_last >= carry_2);
ns_shifted_last.wrapping_sub_assign(carry_2);
}
i_len_alt -= 1;
}
if i_len_alt == 0 {
rs.copy_from_slice(&ns_shifted[..r_len]);
do_extra_cleanup = false;
} else {
let mut mul_scratch = vec![0; limbs_mul_to_out_scratch_len(qs_lo.len(), i_len_alt)];
limbs_mul_to_out(&mut scratch, qs_lo, &ds[..i_len_alt], &mut mul_scratch);
}
if do_extra_cleanup {
let (scratch_lo, scratch_hi) = scratch.split_at_mut(i_len_alt);
q_too_large |=
limbs_sub_greater_in_place_left(&mut ns_shifted[..r_len], &scratch_hi[..q_len]);
let (rs_lo, rs_hi) = rs.split_at_mut(i_len_alt);
let rs_hi_len = rs_hi.len();
rs_hi.copy_from_slice(&ns_shifted[..rs_hi_len]);
q_too_large |= limbs_sub_same_length_to_out(rs_lo, &ns[..i_len_alt], scratch_lo)
&& limbs_sub_limb_in_place(&mut rs_hi[..min(rs_hi_len, r_len)], 1);
}
if q_too_large {
assert!(!limbs_sub_limb_in_place(qs, 1));
limbs_slice_add_same_length_in_place_left(rs, ds);
}
}
pub_test! {limbs_div_mod(ns: &[Limb], ds: &[Limb]) -> (Vec<Limb>, Vec<Limb>) {
let d_len = ds.len();
let mut qs = vec![0; ns.len() - d_len + 1];
let mut rs = vec![0; d_len];
limbs_div_mod_to_out(&mut qs, &mut rs, ns, ds);
(qs, rs)
}}
pub_crate_test! {limbs_div_mod_to_out(qs: &mut [Limb], rs: &mut [Limb], ns: &[Limb], ds: &[Limb]) {
let n_len = ns.len();
let d_len = ds.len();
assert!(d_len > 1);
assert!(n_len >= d_len);
assert!(qs.len() > n_len - d_len);
let rs = &mut rs[..d_len];
let ds_last = *ds.last().unwrap();
assert!(ds_last != 0);
if d_len == 2 {
limbs_div_mod_by_two_limb(qs, rs, ns, ds);
} else {
let adjust = ns[n_len - 1] >= ds_last;
let adjusted_n_len = if adjust { n_len + 1 } else { n_len };
if adjusted_n_len < d_len << 1 {
limbs_div_mod_balanced(qs, rs, ns, ds, adjust);
} else {
limbs_div_mod_unbalanced(qs, rs, ns, ds, adjusted_n_len);
}
}
}}
pub_crate_test! {limbs_div_mod_qs_to_out_rs_to_ns(qs: &mut [Limb], ns: &mut [Limb], ds: &[Limb]) {
let ns_copy = ns.to_vec();
limbs_div_mod_to_out(qs, ns, &ns_copy, ds);
}}
impl Natural {
fn div_mod_limb_ref(&self, other: Limb) -> (Self, Limb) {
match (self, other) {
(_, 0) => panic!("division by zero"),
(n, 1) => (n.clone(), 0),
(Self(Small(small)), other) => {
let (q, r) = small.div_rem(other);
(Self(Small(q)), r)
}
(Self(Large(limbs)), other) => {
let (qs, r) = limbs_div_limb_mod(limbs, other);
(Self::from_owned_limbs_asc(qs), r)
}
}
}
pub_test! {div_assign_mod_limb(&mut self, other: Limb) -> Limb {
match (&mut *self, other) {
(_, 0) => panic!("division by zero"),
(_, 1) => 0,
(Natural(Small(small)), other) => small.div_assign_rem(other),
(Natural(Large(limbs)), other) => {
let r = limbs_div_limb_in_place_mod(limbs, other);
self.trim();
r
}
}
}}
}
impl DivMod<Self> for Natural {
type DivOutput = Self;
type ModOutput = Self;
#[inline]
fn div_mod(mut self, other: Self) -> (Self, Self) {
let r = self.div_assign_mod(other);
(self, r)
}
}
impl<'a> DivMod<&'a Self> for Natural {
type DivOutput = Self;
type ModOutput = Self;
#[inline]
fn div_mod(mut self, other: &'a Self) -> (Self, Self) {
let r = self.div_assign_mod(other);
(self, r)
}
}
impl DivMod<Natural> for &Natural {
type DivOutput = Natural;
type ModOutput = Natural;
fn div_mod(self, mut other: Natural) -> (Natural, Natural) {
if *self == other {
return (Natural::ONE, Natural::ZERO);
}
match (self, &mut other) {
(_, &mut Natural::ZERO) => panic!("division by zero"),
(n, &mut Natural::ONE) => (n.clone(), Natural::ZERO),
(n, &mut Natural(Small(d))) => {
let (q, r) = n.div_mod_limb_ref(d);
(q, Natural(Small(r)))
}
(Natural(Small(_)), _) => (Natural::ZERO, self.clone()),
(Natural(Large(ns)), Natural(Large(ds))) => {
if ns.len() < ds.len() {
(Natural::ZERO, self.clone())
} else {
let (qs, mut rs) = limbs_div_mod(ns, ds);
swap(&mut rs, ds);
other.trim();
(Natural::from_owned_limbs_asc(qs), other)
}
}
}
}
}
impl DivMod<&Natural> for &Natural {
type DivOutput = Natural;
type ModOutput = Natural;
fn div_mod(self, other: &Natural) -> (Natural, Natural) {
if self == other {
return (Natural::ONE, Natural::ZERO);
}
match (self, other) {
(_, &Natural::ZERO) => panic!("division by zero"),
(n, &Natural::ONE) => (n.clone(), Natural::ZERO),
(n, Natural(Small(d))) => {
let (q, r) = n.div_mod_limb_ref(*d);
(q, Natural(Small(r)))
}
(Natural(Small(_)), _) => (Natural::ZERO, self.clone()),
(Natural(Large(ns)), Natural(Large(ds))) => {
if ns.len() < ds.len() {
(Natural::ZERO, self.clone())
} else {
let (qs, rs) = limbs_div_mod(ns, ds);
(
Natural::from_owned_limbs_asc(qs),
Natural::from_owned_limbs_asc(rs),
)
}
}
}
}
}
impl DivAssignMod<Self> for Natural {
type ModOutput = Self;
fn div_assign_mod(&mut self, mut other: Self) -> Self {
if *self == other {
*self = Self::ONE;
return Self::ZERO;
}
match (&mut *self, &mut other) {
(_, &mut Self::ZERO) => panic!("division by zero"),
(_, &mut Self::ONE) => Self::ZERO,
(n, &mut Self(Small(d))) => Self(Small(n.div_assign_mod_limb(d))),
(Self(Small(_)), _) => {
let mut r = Self::ZERO;
swap(self, &mut r);
r
}
(Self(Large(ns)), Self(Large(ds))) => {
if ns.len() < ds.len() {
let mut r = Self::ZERO;
swap(self, &mut r);
r
} else {
let (mut qs, mut rs) = limbs_div_mod(ns, ds);
swap(&mut qs, ns);
swap(&mut rs, ds);
self.trim();
other.trim();
other
}
}
}
}
}
impl<'a> DivAssignMod<&'a Self> for Natural {
type ModOutput = Self;
fn div_assign_mod(&mut self, other: &'a Self) -> Self {
if self == other {
*self = Self::ONE;
return Self::ZERO;
}
match (&mut *self, other) {
(_, &Self::ZERO) => panic!("division by zero"),
(_, &Self::ONE) => Self::ZERO,
(_, Self(Small(d))) => Self(Small(self.div_assign_mod_limb(*d))),
(Self(Small(_)), _) => {
let mut r = Self::ZERO;
swap(self, &mut r);
r
}
(Self(Large(ns)), Self(Large(ds))) => {
if ns.len() < ds.len() {
let mut r = Self::ZERO;
swap(self, &mut r);
r
} else {
let (mut qs, rs) = limbs_div_mod(ns, ds);
swap(&mut qs, ns);
self.trim();
Self::from_owned_limbs_asc(rs)
}
}
}
}
}
impl DivRem<Self> for Natural {
type DivOutput = Self;
type RemOutput = Self;
#[inline]
fn div_rem(self, other: Self) -> (Self, Self) {
self.div_mod(other)
}
}
impl<'a> DivRem<&'a Self> for Natural {
type DivOutput = Self;
type RemOutput = Self;
#[inline]
fn div_rem(self, other: &'a Self) -> (Self, Self) {
self.div_mod(other)
}
}
impl DivRem<Natural> for &Natural {
type DivOutput = Natural;
type RemOutput = Natural;
#[inline]
fn div_rem(self, other: Natural) -> (Natural, Natural) {
self.div_mod(other)
}
}
impl DivRem<&Natural> for &Natural {
type DivOutput = Natural;
type RemOutput = Natural;
#[inline]
fn div_rem(self, other: &Natural) -> (Natural, Natural) {
self.div_mod(other)
}
}
impl DivAssignRem<Self> for Natural {
type RemOutput = Self;
#[inline]
fn div_assign_rem(&mut self, other: Self) -> Self {
self.div_assign_mod(other)
}
}
impl<'a> DivAssignRem<&'a Self> for Natural {
type RemOutput = Self;
#[inline]
fn div_assign_rem(&mut self, other: &'a Self) -> Self {
self.div_assign_mod(other)
}
}
impl CeilingDivNegMod<Self> for Natural {
type DivOutput = Self;
type ModOutput = Self;
#[inline]
fn ceiling_div_neg_mod(mut self, other: Self) -> (Self, Self) {
let r = self.ceiling_div_assign_neg_mod(other);
(self, r)
}
}
impl<'a> CeilingDivNegMod<&'a Self> for Natural {
type DivOutput = Self;
type ModOutput = Self;
#[inline]
fn ceiling_div_neg_mod(mut self, other: &'a Self) -> (Self, Self) {
let r = self.ceiling_div_assign_neg_mod(other);
(self, r)
}
}
impl CeilingDivNegMod<Natural> for &Natural {
type DivOutput = Natural;
type ModOutput = Natural;
fn ceiling_div_neg_mod(self, other: Natural) -> (Natural, Natural) {
let (q, r) = self.div_mod(&other);
if r == 0 {
(q, r)
} else {
(q.add_limb(1), other - r)
}
}
}
impl CeilingDivNegMod<&Natural> for &Natural {
type DivOutput = Natural;
type ModOutput = Natural;
fn ceiling_div_neg_mod(self, other: &Natural) -> (Natural, Natural) {
let (q, r) = self.div_mod(other);
if r == 0 {
(q, r)
} else {
(q.add_limb(1), other - r)
}
}
}
impl CeilingDivAssignNegMod<Self> for Natural {
type ModOutput = Self;
fn ceiling_div_assign_neg_mod(&mut self, other: Self) -> Self {
let r = self.div_assign_mod(&other);
if r == 0 {
Self::ZERO
} else {
*self += Self::ONE;
other - r
}
}
}
impl<'a> CeilingDivAssignNegMod<&'a Self> for Natural {
type ModOutput = Self;
fn ceiling_div_assign_neg_mod(&mut self, other: &'a Self) -> Self {
let r = self.div_assign_mod(other);
if r == 0 {
Self::ZERO
} else {
*self += Self::ONE;
other - r
}
}
}