use core::cmp;
use core::mem::{self, MaybeUninit};
use core::ptr;
struct CopyOnDrop<T> {
src: *mut T,
dest: *mut T,
}
impl<T> Drop for CopyOnDrop<T> {
fn drop(&mut self) {
unsafe {
ptr::copy_nonoverlapping(self.src, self.dest, 1);
}
}
}
fn shift_head<T, F>(v: &mut [T], is_less: &mut F) -> Option<()>
where
F: FnMut(&T, &T) -> Option<bool>,
{
let len = v.len();
unsafe {
if len >= 2 && is_less(v.get_unchecked(1), v.get_unchecked(0))? {
let mut tmp = mem::ManuallyDrop::new(ptr::read(v.get_unchecked(0)));
let mut hole = CopyOnDrop {
src: &mut *tmp,
dest: v.get_unchecked_mut(1),
};
ptr::copy_nonoverlapping(v.get_unchecked(1), v.get_unchecked_mut(0), 1);
for i in 2..len {
if !is_less(v.get_unchecked(i), &*tmp)? {
break;
}
ptr::copy_nonoverlapping(v.get_unchecked(i), v.get_unchecked_mut(i - 1), 1);
hole.dest = v.get_unchecked_mut(i);
}
}
Some(())
}
}
fn shift_tail<T, F>(v: &mut [T], is_less: &mut F) -> Option<()>
where
F: FnMut(&T, &T) -> Option<bool>,
{
let len = v.len();
unsafe {
if len >= 2 && is_less(v.get_unchecked(len - 1), v.get_unchecked(len - 2))? {
let mut tmp = mem::ManuallyDrop::new(ptr::read(v.get_unchecked(len - 1)));
let mut hole = CopyOnDrop {
src: &mut *tmp,
dest: v.get_unchecked_mut(len - 2),
};
ptr::copy_nonoverlapping(v.get_unchecked(len - 2), v.get_unchecked_mut(len - 1), 1);
for i in (0..len - 2).rev() {
if !is_less(&*tmp, v.get_unchecked(i))? {
break;
}
ptr::copy_nonoverlapping(v.get_unchecked(i), v.get_unchecked_mut(i + 1), 1);
hole.dest = v.get_unchecked_mut(i);
}
}
Some(())
}
}
#[cold]
fn partial_insertion_sort<T, F>(v: &mut [T], is_less: &mut F) -> Option<bool>
where
F: FnMut(&T, &T) -> Option<bool>,
{
const MAX_STEPS: usize = 5;
const SHORTEST_SHIFTING: usize = 50;
let len = v.len();
let mut i = 1;
for _ in 0..MAX_STEPS {
unsafe {
while i < len && !is_less(v.get_unchecked(i), v.get_unchecked(i - 1))? {
i += 1;
}
}
if i == len {
return Some(true);
}
if len < SHORTEST_SHIFTING {
return Some(false);
}
v.swap(i - 1, i);
shift_tail(&mut v[..i], is_less);
shift_head(&mut v[i..], is_less);
}
Some(false)
}
fn insertion_sort<T, F>(v: &mut [T], is_less: &mut F) -> Option<()>
where
F: FnMut(&T, &T) -> Option<bool>,
{
for i in 1..v.len() {
shift_tail(&mut v[..i + 1], is_less)?;
}
Some(())
}
#[cold]
pub fn heapsort<T, F>(v: &mut [T], mut is_less: F) -> Option<()>
where
F: FnMut(&T, &T) -> Option<bool>,
{
let mut sift_down = |v: &mut [T], mut node: usize| {
loop {
let left = 2 * node + 1;
let right = 2 * node + 2;
let greater = if right < v.len() {
if let Some(lt) = is_less(&v[left], &v[right]) {
if lt {
right
} else {
left
}
} else {
return None;
}
} else {
left
};
if greater >= v.len() || !is_less(&v[node], &v[greater])? {
break Some(());
}
v.swap(node, greater);
node = greater;
}
};
for i in (0..v.len() / 2).rev() {
sift_down(v, i)?;
}
for i in (1..v.len()).rev() {
v.swap(0, i);
sift_down(&mut v[..i], 0);
}
Some(())
}
fn partition_in_blocks<T, F>(v: &mut [T], pivot: &T, is_less: &mut F) -> Option<usize>
where
F: FnMut(&T, &T) -> Option<bool>,
{
const BLOCK: usize = 128;
let mut l = v.as_mut_ptr();
let mut block_l = BLOCK;
let mut start_l = ptr::null_mut();
let mut end_l = ptr::null_mut();
let mut offsets_l = [MaybeUninit::<u8>::uninit(); BLOCK];
let mut r = unsafe { l.add(v.len()) };
let mut block_r = BLOCK;
let mut start_r = ptr::null_mut();
let mut end_r = ptr::null_mut();
let mut offsets_r = [MaybeUninit::<u8>::uninit(); BLOCK];
fn width<T>(l: *mut T, r: *mut T) -> usize {
assert!(mem::size_of::<T>() > 0);
(r as usize - l as usize) / mem::size_of::<T>()
}
loop {
let is_done = width(l, r) <= 2 * BLOCK;
if is_done {
let mut rem = width(l, r);
if start_l < end_l || start_r < end_r {
rem -= BLOCK;
}
if start_l < end_l {
block_r = rem;
} else if start_r < end_r {
block_l = rem;
} else {
block_l = rem / 2;
block_r = rem - block_l;
}
debug_assert!(block_l <= BLOCK && block_r <= BLOCK);
debug_assert!(width(l, r) == block_l + block_r);
}
fn slice_as_mut_ptr<T>(this: &mut [MaybeUninit<T>]) -> *mut T {
this.as_mut_ptr() as *mut T
}
if start_l == end_l {
start_l = slice_as_mut_ptr(&mut offsets_l);
end_l = slice_as_mut_ptr(&mut offsets_l);
let mut elem = l;
for i in 0..block_l {
unsafe {
*end_l = i as u8;
end_l = end_l.offset(!is_less(&*elem, pivot)? as isize);
elem = elem.offset(1);
}
}
}
if start_r == end_r {
start_r = slice_as_mut_ptr(&mut offsets_r);
end_r = slice_as_mut_ptr(&mut offsets_r);
let mut elem = r;
for i in 0..block_r {
unsafe {
elem = elem.offset(-1);
*end_r = i as u8;
end_r = end_r.offset(is_less(&*elem, pivot)? as isize);
}
}
}
let count = cmp::min(width(start_l, end_l), width(start_r, end_r));
if count > 0 {
macro_rules! left {
() => {
l.offset(*start_l as isize)
};
}
macro_rules! right {
() => {
r.offset(-(*start_r as isize) - 1)
};
}
unsafe {
let tmp = ptr::read(left!());
ptr::copy_nonoverlapping(right!(), left!(), 1);
for _ in 1..count {
start_l = start_l.offset(1);
ptr::copy_nonoverlapping(left!(), right!(), 1);
start_r = start_r.offset(1);
ptr::copy_nonoverlapping(right!(), left!(), 1);
}
ptr::copy_nonoverlapping(&tmp, right!(), 1);
mem::forget(tmp);
start_l = start_l.offset(1);
start_r = start_r.offset(1);
}
}
if start_l == end_l {
l = unsafe { l.offset(block_l as isize) };
}
if start_r == end_r {
r = unsafe { r.offset(-(block_r as isize)) };
}
if is_done {
break;
}
}
if start_l < end_l {
debug_assert_eq!(width(l, r), block_l);
while start_l < end_l {
unsafe {
end_l = end_l.offset(-1);
ptr::swap(l.offset(*end_l as isize), r.offset(-1));
r = r.offset(-1);
}
}
Some(width(v.as_mut_ptr(), r))
} else if start_r < end_r {
debug_assert_eq!(width(l, r), block_r);
while start_r < end_r {
unsafe {
end_r = end_r.offset(-1);
ptr::swap(l, r.offset(-(*end_r as isize) - 1));
l = l.offset(1);
}
}
Some(width(v.as_mut_ptr(), l))
} else {
Some(width(v.as_mut_ptr(), l))
}
}
fn partition<T, F>(v: &mut [T], pivot: usize, is_less: &mut F) -> Option<(usize, bool)>
where
F: FnMut(&T, &T) -> Option<bool>,
{
let (mid, was_partitioned) = {
v.swap(0, pivot);
let (pivot, v) = v.split_at_mut(1);
let pivot = &mut pivot[0];
let mut tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) });
let _pivot_guard = CopyOnDrop {
src: &mut *tmp,
dest: pivot,
};
let pivot = &*tmp;
let mut l = 0;
let mut r = v.len();
unsafe {
while l < r && is_less(v.get_unchecked(l), pivot)? {
l += 1;
}
while l < r && !is_less(v.get_unchecked(r - 1), pivot)? {
r -= 1;
}
}
(
l + partition_in_blocks(&mut v[l..r], pivot, is_less)?,
l >= r,
)
};
v.swap(0, mid);
Some((mid, was_partitioned))
}
fn partition_equal<T, F>(v: &mut [T], pivot: usize, is_less: &mut F) -> Option<usize>
where
F: FnMut(&T, &T) -> Option<bool>,
{
v.swap(0, pivot);
let (pivot, v) = v.split_at_mut(1);
let pivot = &mut pivot[0];
let mut tmp = mem::ManuallyDrop::new(unsafe { ptr::read(pivot) });
let _pivot_guard = CopyOnDrop {
src: &mut *tmp,
dest: pivot,
};
let pivot = &*tmp;
let mut l = 0;
let mut r = v.len();
loop {
unsafe {
while l < r && !is_less(pivot, v.get_unchecked(l))? {
l += 1;
}
while l < r && is_less(pivot, v.get_unchecked(r - 1))? {
r -= 1;
}
if l >= r {
break;
}
r -= 1;
ptr::swap(v.get_unchecked_mut(l), v.get_unchecked_mut(r));
l += 1;
}
}
Some(l + 1)
}
#[cold]
fn break_patterns<T>(v: &mut [T]) {
let len = v.len();
if len >= 8 {
let mut random = len as u32;
let mut gen_u32 = || {
random ^= random << 13;
random ^= random >> 17;
random ^= random << 5;
random
};
let mut gen_usize = || {
if mem::size_of::<usize>() <= 4 {
gen_u32() as usize
} else {
(((gen_u32() as u64) << 32) | (gen_u32() as u64)) as usize
}
};
let modulus = len.next_power_of_two();
let pos = len / 4 * 2;
for i in 0..3 {
let mut other = gen_usize() & (modulus - 1);
if other >= len {
other -= len;
}
v.swap(pos - 1 + i, other);
}
}
}
fn choose_pivot<T, F>(v: &mut [T], is_less: &mut F) -> Option<(usize, bool)>
where
F: FnMut(&T, &T) -> Option<bool>,
{
const SHORTEST_MEDIAN_OF_MEDIANS: usize = 50;
const MAX_SWAPS: usize = 4 * 3;
let len = v.len();
let mut a = len / 4 * 1;
let mut b = len / 4 * 2;
let mut c = len / 4 * 3;
let mut swaps = 0;
if len >= 8 {
let mut sort2 = |a: &mut usize, b: &mut usize| unsafe {
if is_less(v.get_unchecked(*b), v.get_unchecked(*a))? {
ptr::swap(a, b);
swaps += 1;
}
Some(())
};
let mut sort3 = |a: &mut usize, b: &mut usize, c: &mut usize| {
sort2(a, b).and(sort2(b, c)).and(sort2(a, b))
};
if len >= SHORTEST_MEDIAN_OF_MEDIANS {
let mut sort_adjacent = |a: &mut usize| {
let tmp = *a;
sort3(&mut (tmp - 1), a, &mut (tmp + 1))
};
sort_adjacent(&mut a)?;
sort_adjacent(&mut b)?;
sort_adjacent(&mut c)?;
}
sort3(&mut a, &mut b, &mut c)?;
}
Some(if swaps < MAX_SWAPS {
(b, swaps == 0)
} else {
v.reverse();
(len - 1 - b, true)
})
}
fn recurse<'a, T, F>(
mut v: &'a mut [T],
is_less: &mut F,
mut pred: Option<&'a T>,
mut limit: u32,
) -> Option<()>
where
F: FnMut(&T, &T) -> Option<bool>,
{
const MAX_INSERTION: usize = 20;
let mut was_balanced = true;
let mut was_partitioned = true;
loop {
let len = v.len();
if len <= MAX_INSERTION {
insertion_sort(v, is_less);
return Some(());
}
if limit == 0 {
heapsort(v, is_less);
return Some(());
}
if !was_balanced {
break_patterns(v);
limit -= 1;
}
let (pivot, likely_sorted) = choose_pivot(v, is_less)?;
if was_balanced && was_partitioned && likely_sorted {
if partial_insertion_sort(v, is_less)? {
return Some(());
}
}
if let Some(p) = pred {
if !is_less(p, &v[pivot])? {
let mid = partition_equal(v, pivot, is_less)?;
v = &mut { v }[mid..];
continue;
}
}
let (mid, was_p) = partition(v, pivot, is_less)?;
was_balanced = cmp::min(mid, len - mid) >= len / 8;
was_partitioned = was_p;
let (left, right) = { v }.split_at_mut(mid);
let (pivot, right) = right.split_at_mut(1);
let pivot = &pivot[0];
if left.len() < right.len() {
recurse(left, is_less, pred, limit);
v = right;
pred = Some(pivot);
} else {
recurse(right, is_less, Some(pivot), limit);
v = left;
}
}
}
pub fn quicksort<T, F>(v: &mut [T], mut is_less: F) -> Option<()>
where
F: FnMut(&T, &T) -> Option<bool>,
{
if mem::size_of::<T>() == 0 {
return Some(());
}
let limit = mem::size_of::<usize>() as u32 * 8 - v.len().leading_zeros();
recurse(v, &mut is_less, None, limit)
}