use super::bitmap_manager::{normalize_bit_count_offsets, process_negative_offset, reverse};
#[inline]
pub(crate) fn bit_index_count(payload: u8, start_bit_offset: u32, end_bit_offset: u32) -> i64 {
let payload = reverse(payload);
let left_bit_index = 1u16 << start_bit_offset;
let right_bit_index = 1u16 << end_bit_offset;
let mask = (right_bit_index - left_bit_index) as u8;
(mask & payload).count_ones() as i64
}
fn bit_index_count_range(value: &[u8], start_offset: i64, end_offset: i64) -> i64 {
let start_byte = start_offset / 8;
let end_byte = end_offset / 8;
let left_bit_index = (start_offset & 7) as u32;
let right_bit_index = (end_offset & 7) as u32 + 1;
if start_byte == end_byte {
bit_index_count(value[start_byte as usize], left_bit_index, right_bit_index)
} else {
bit_index_count(value[start_byte as usize], left_bit_index, 8)
+ bit_index_count(value[end_byte as usize], 0, right_bit_index)
}
}
pub fn bit_count_driver(
mut start_offset: i64,
mut end_offset: i64,
offset_type: u8,
value: &[u8],
val_len: i64,
) -> i64 {
let mut count = 0;
(start_offset, end_offset) = normalize_bit_count_offsets(start_offset, end_offset, offset_type);
if offset_type == 0x0 {
if start_offset < 0 {
start_offset = process_negative_offset(start_offset, val_len);
}
if end_offset < 0 {
end_offset = process_negative_offset(end_offset, val_len);
}
if end_offset >= val_len {
end_offset = val_len - 1;
}
if start_offset >= val_len {
return 0;
}
if start_offset > end_offset {
return 0;
}
} else {
let bit_len = val_len * 8;
if bit_len == 0 {
return 0;
}
if start_offset < 0 {
start_offset = process_negative_offset(start_offset, bit_len);
}
if end_offset < 0 {
end_offset = process_negative_offset(end_offset, bit_len);
}
if start_offset >= bit_len {
return 0;
}
if start_offset > end_offset {
return 0;
}
if end_offset >= bit_len {
end_offset = bit_len - 1;
}
count += bit_index_count_range(value, start_offset, end_offset);
start_offset = (start_offset / 8) + 1;
end_offset = (end_offset / 8) - 1;
if start_offset >= end_offset {
return count;
}
}
let (start, end) = (start_offset as usize, end_offset as usize);
if end - start < 128 {
count += __scalar_popc(value, start, end);
} else {
count += __simd_popc_x256(value, start, end);
}
count
}
pub fn __scalar_popc(bitmap: &[u8], start: usize, end: usize) -> i64 {
let mut count: u64 = 0;
let mut batch_size: usize = 8 * 4;
let mut len = (end - start) + 1;
let mut tail = len & (batch_size - 1);
let mut curr = start;
let mut vend = curr + (len - (len & tail));
while curr < vend {
let v00 = u64_read(bitmap, curr).count_ones() as u64;
let v01 = u64_read(bitmap, curr + 8).count_ones() as u64;
let v02 = u64_read(bitmap, curr + 16).count_ones() as u64;
let v03 = u64_read(bitmap, curr + 24).count_ones() as u64;
count += (v00 + v01) + (v02 + v03);
curr += batch_size;
}
len = tail;
batch_size = 8;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += u64_read(bitmap, curr).count_ones() as u64;
curr += batch_size;
}
count += popc_tail(bitmap, curr, tail) as u64;
count as i64
}
pub fn __simd_popc_x128(bitmap: &[u8], start: usize, end: usize) -> i64 {
let mut count: u64 = 0;
let mut batch_size: usize = 8 * 16;
let mut len = (end - start) + 1;
let mut tail = len & (batch_size - 1);
let mut curr = start;
let mut vend = curr + (len - (len & tail));
while curr < vend {
count += popc_bytes(bitmap, curr, 8 * 16);
curr += batch_size;
}
if tail == 0 {
return count as i64;
}
len = tail;
batch_size = 16;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += popc_bytes(bitmap, curr, 16);
curr += batch_size;
}
if tail == 0 {
return count as i64;
}
len = tail;
batch_size = 4 * 8;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += u64_read(bitmap, curr).count_ones() as u64
+ u64_read(bitmap, curr + 8).count_ones() as u64
+ u64_read(bitmap, curr + 16).count_ones() as u64
+ u64_read(bitmap, curr + 24).count_ones() as u64;
curr += batch_size;
}
if tail == 0 {
return count as i64;
}
len = tail;
batch_size = 8;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += u64_read(bitmap, curr).count_ones() as u64;
curr += 8;
}
if tail == 0 {
return count as i64;
}
count += popc_tail(bitmap, curr, tail) as u64;
count as i64
}
pub fn __simd_popc_x256(bitmap: &[u8], start: usize, end: usize) -> i64 {
let mut count: u64 = 0;
let mut batch_size: usize = 8 * 32;
let mut len = (end - start) + 1;
let mut tail = len & (batch_size - 1);
let mut curr = start;
let mut vend = curr + (len - (len & tail));
while curr < vend {
count += popc_bytes(bitmap, curr, 8 * 32);
curr += batch_size;
}
if tail == 0 {
return count as i64;
}
len = tail;
batch_size = 32;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += popc_bytes(bitmap, curr, 32);
curr += batch_size;
}
if tail == 0 {
return count as i64;
}
len = tail;
batch_size = 4 * 8;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += u64_read(bitmap, curr).count_ones() as u64
+ u64_read(bitmap, curr + 8).count_ones() as u64
+ u64_read(bitmap, curr + 16).count_ones() as u64
+ u64_read(bitmap, curr + 24).count_ones() as u64;
curr += batch_size;
}
if tail == 0 {
return count as i64;
}
len = tail;
batch_size = 8;
tail = len & (batch_size - 1);
vend = curr + (len - (len & tail));
while curr < vend {
count += u64_read(bitmap, curr).count_ones() as u64;
curr += 8;
}
if tail == 0 {
return count as i64;
}
count += popc_tail(bitmap, curr, tail) as u64;
count as i64
}
#[inline]
fn u64_read(bitmap: &[u8], idx: usize) -> u64 {
u64::from_le_bytes(bitmap[idx..idx + 8].try_into().unwrap())
}
#[inline]
fn popc_bytes(bitmap: &[u8], idx: usize, width: usize) -> u64 {
bitmap[idx..idx + width]
.as_chunks::<8>()
.0
.iter()
.map(|c| u64::from_le_bytes(*c).count_ones() as u64)
.sum()
}
#[inline]
fn popc_tail(bitmap: &[u8], idx: usize, tail: usize) -> u32 {
let mut tt: u64 = 0;
if tail >= 7 {
tt |= u64::from(bitmap[idx + 6]) << 48;
}
if tail >= 6 {
tt |= u64::from(bitmap[idx + 5]) << 40;
}
if tail >= 5 {
tt |= u64::from(bitmap[idx + 4]) << 32;
}
if tail >= 4 {
tt |= u64::from(bitmap[idx + 3]) << 24;
}
if tail >= 3 {
tt |= u64::from(bitmap[idx + 2]) << 16;
}
if tail >= 2 {
tt |= u64::from(bitmap[idx + 1]) << 8;
}
if tail >= 1 {
tt |= u64::from(bitmap[idx]);
}
tt.count_ones()
}
#[cfg(test)]
mod tests {
use super::{__scalar_popc, __simd_popc_x128, __simd_popc_x256, bit_count_driver};
fn pseudo_random(len: usize) -> Vec<u8> {
let mut x: u64 = 0x2545_F491_4F6C_DD1D;
(0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x as u8
})
.collect()
}
fn naive(bitmap: &[u8], start: usize, end: usize) -> i64 {
bitmap[start..=end]
.iter()
.map(|b| b.count_ones() as i64)
.sum()
}
#[test]
fn popc_variants_match_naive() {
let bitmap = pseudo_random(1024);
for &(s, e) in &[
(0, 0),
(0, 7),
(0, 8),
(5, 12),
(0, 31),
(3, 130),
(0, 255),
(1, 256),
(0, 511),
(7, 1023),
(13, 777),
] {
let want = naive(&bitmap, s, e);
assert_eq!(__scalar_popc(&bitmap, s, e), want, "scalar [{s},{e}]");
assert_eq!(__simd_popc_x128(&bitmap, s, e), want, "x128 [{s},{e}]");
assert_eq!(__simd_popc_x256(&bitmap, s, e), want, "x256 [{s},{e}]");
}
}
#[test]
fn driver_byte_and_bit_modes() {
let val = [0xA5u8, 0x0F];
assert_eq!(bit_count_driver(0, -1, 0x0, &val, 2), 8);
assert_eq!(bit_count_driver(0, 0, 0x0, &val, 2), 4);
assert_eq!(bit_count_driver(-1, -1, 0x0, &val, 2), 4);
assert_eq!(bit_count_driver(2, 100, 0x0, &val, 2), 0);
assert_eq!(bit_count_driver(0, 7, 0x1, &val, 2), 4);
assert_eq!(bit_count_driver(1, 2, 0x1, &val, 2), 1);
assert_eq!(bit_count_driver(3, 4, 0x1, &val, 2), 0);
assert_eq!(bit_count_driver(8, 15, 0x1, &val, 2), 4);
assert_eq!(bit_count_driver(16, 23, 0x1, &val, 2), 0);
assert_eq!(bit_count_driver(0, -1, 0x1, &val, 0), 0);
}
}