use crossbeam_epoch::{self as epoch};
use log::{log_enabled, trace, error};
use routecore::record::{MergeUpdate, Meta};
use std::hash::Hash;
use std::sync::atomic::{
AtomicU16, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering,
};
use std::{fmt::Debug, marker::PhantomData};
use crate::af::AddressFamily;
use crate::custom_alloc::CustomAllocStorage;
use crate::insert_match;
use crate::local_array::store::atomic_types::{NodeBuckets, PrefixBuckets};
use crate::prefix_record::InternalPrefixRecord;
pub(crate) use super::atomic_stride::*;
use super::store::errors::PrefixStoreError;
use crate::stats::{SizedStride, StrideStats};
pub(crate) use crate::local_array::node::TreeBitMapNode;
#[cfg(feature = "cli")]
use ansi_term::Colour;
#[allow(clippy::large_enum_variant)]
#[derive(Debug)]
pub enum SizedStrideNode<AF: AddressFamily> {
Stride3(TreeBitMapNode<AF, Stride3>),
Stride4(TreeBitMapNode<AF, Stride4>),
Stride5(TreeBitMapNode<AF, Stride5>),
}
impl<AF, S> Default for TreeBitMapNode<AF, S>
where
AF: AddressFamily,
S: Stride,
{
fn default() -> Self {
Self {
ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
_af: PhantomData,
}
}
}
impl<AF> Default for SizedStrideNode<AF>
where
AF: AddressFamily,
{
fn default() -> Self {
SizedStrideNode::Stride3(TreeBitMapNode {
ptrbitarr: AtomicStride2(AtomicU8::new(0)),
pfxbitarr: AtomicStride3(AtomicU16::new(0)),
_af: PhantomData,
})
}
}
#[derive(Debug)]
pub enum SizedStrideRef<'a, AF: AddressFamily> {
Stride3(&'a TreeBitMapNode<AF, Stride3>),
Stride4(&'a TreeBitMapNode<AF, Stride4>),
Stride5(&'a TreeBitMapNode<AF, Stride5>),
}
#[derive(Debug)]
pub enum SizedStrideRefMut<'a, AF: AddressFamily> {
Stride3(&'a mut TreeBitMapNode<AF, Stride3>),
Stride4(&'a mut TreeBitMapNode<AF, Stride4>),
Stride5(&'a mut TreeBitMapNode<AF, Stride5>),
}
pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
NewNode(SizedStrideNode<AF>),
ExistingNode(StrideNodeId<AF>),
NewPrefix,
ExistingPrefix,
}
#[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
pub struct PrefixId<AF: AddressFamily>(Option<(AF, u8)>);
impl<AF: AddressFamily> PrefixId<AF> {
pub fn new(net: AF, len: u8) -> Self {
PrefixId(Some((net, len)))
}
pub fn is_empty(&self) -> bool {
self.0.is_none()
}
pub fn get_net(&self) -> AF {
self.0.unwrap().0
}
pub fn get_len(&self) -> u8 {
self.0.unwrap().1
}
pub fn into_pub(&self) -> routecore::addr::Prefix {
routecore::addr::Prefix::new(
self.get_net().into_ipaddr(),
self.get_len(),
)
.unwrap_or_else(|p| panic!("can't convert {:?} into prefix.", p))
}
pub fn inc_len(self) -> Self {
Self(self.0.map(|(net, len)| (net, len + 1)))
}
}
impl<AF: AddressFamily> std::default::Default for PrefixId<AF> {
fn default() -> Self {
PrefixId(None)
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub struct StrideNodeId<AF: AddressFamily>(Option<(AF, u8)>);
impl<AF: AddressFamily> StrideNodeId<AF> {
pub fn empty() -> Self {
Self(None)
}
pub fn dangerously_new_with_id_as_is(addr_bits: AF, len: u8) -> Self {
Self(Some((addr_bits, len)))
}
#[inline]
pub fn new_with_cleaned_id(addr_bits: AF, len: u8) -> Self {
Self(Some((addr_bits.truncate_to_len(len), len)))
}
pub fn is_empty(&self) -> bool {
self.0.is_none()
}
pub fn get_id(&self) -> (AF, u8) {
self.0.unwrap()
}
pub fn get_len(&self) -> u8 {
self.0.unwrap().1
}
pub fn set_len(mut self, len: u8) -> Self {
self.0.as_mut().unwrap().1 = len;
self
}
pub fn add_to_len(mut self, len: u8) -> Self {
self.0.as_mut().unwrap().1 += len;
self
}
#[inline]
pub fn truncate_to_len(self) -> Self {
let (addr_bits, len) = self.0.unwrap();
StrideNodeId::new_with_cleaned_id(addr_bits, len)
}
#[inline]
pub fn unwrap_with_cleaned_id(&self) -> (AF, u8) {
let (addr_bits, len) = self.0.unwrap();
(addr_bits.truncate_to_len(len), len)
}
pub fn add_nibble(&self, nibble: u32, nibble_len: u8) -> Self {
let (addr_bits, len) = self.unwrap_with_cleaned_id();
let res = addr_bits.add_nibble(len, nibble, nibble_len);
Self(Some(res))
}
pub fn into_inner(self) -> Option<(AF, u8)> {
self.0
}
}
impl<AF: AddressFamily> std::fmt::Display for StrideNodeId<AF> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
self.0
.map(|x| format!("{}-{}", x.0, x.1))
.unwrap_or_else(|| "-".to_string())
)
}
}
impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
for PrefixId<AF>
{
fn from(id: StrideNodeId<AF>) -> Self {
let (addr_bits, len) = id.0.unwrap();
PrefixId::new(addr_bits, len)
}
}
#[derive(Debug)]
pub struct AtomicStrideNodeId<AF: AddressFamily> {
stride_type: StrideType,
index: AtomicU32,
serial: AtomicUsize,
_af: PhantomData<AF>,
}
impl<AF: AddressFamily> AtomicStrideNodeId<AF> {
pub fn new(stride_type: StrideType, index: u32) -> Self {
Self {
stride_type,
index: AtomicU32::new(index),
serial: AtomicUsize::new(1),
_af: PhantomData,
}
}
pub fn empty() -> Self {
Self {
stride_type: StrideType::Stride4,
index: AtomicU32::new(0),
serial: AtomicUsize::new(0),
_af: PhantomData,
}
}
pub fn get_serial(&self) -> usize {
let serial = self.serial.load(Ordering::SeqCst);
std::sync::atomic::fence(Ordering::SeqCst);
serial
}
pub fn update_serial(
&self,
current_serial: usize,
) -> Result<usize, usize> {
std::sync::atomic::fence(Ordering::Release);
self.serial.compare_exchange(
current_serial,
current_serial + 1,
Ordering::SeqCst,
Ordering::SeqCst,
)
}
pub fn set_id(&self, index: u32) -> Result<u32, u32> {
self.index.compare_exchange(
0,
index,
Ordering::SeqCst,
Ordering::SeqCst,
)
}
pub fn is_empty(&self) -> bool {
self.serial.load(Ordering::SeqCst) == 0
}
pub fn into_inner(self) -> (StrideType, Option<u32>) {
match self.serial.load(Ordering::SeqCst) {
0 => (self.stride_type, None),
_ => (
self.stride_type,
Some(self.index.load(Ordering::SeqCst) as u32),
),
}
}
pub fn from_stridenodeid(
stride_type: StrideType,
id: StrideNodeId<AF>,
) -> Self {
let index: AF = id.0.map_or(AF::zero(), |i| i.0);
Self {
stride_type,
index: AtomicU32::new(index.dangerously_truncate_to_u32()),
serial: AtomicUsize::new(if index == AF::zero() { 0 } else { 1 }),
_af: PhantomData,
}
}
}
impl<AF: AddressFamily> std::convert::From<AtomicStrideNodeId<AF>> for usize {
fn from(id: AtomicStrideNodeId<AF>) -> Self {
id.index.load(Ordering::SeqCst) as usize
}
}
pub trait NodeCollection<AF: AddressFamily> {
fn insert(&mut self, index: u16, insert_node: StrideNodeId<AF>);
fn to_vec(&self) -> Vec<StrideNodeId<AF>>;
fn as_slice(&self) -> &[AtomicStrideNodeId<AF>];
fn empty() -> Self;
}
#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Clone)]
pub enum StrideType {
Stride3,
Stride4,
Stride5,
}
impl From<u8> for StrideType {
fn from(level: u8) -> Self {
match level {
3 => StrideType::Stride3,
4 => StrideType::Stride4,
5 => StrideType::Stride5,
_ => panic!("Invalid stride level {}", level),
}
}
}
impl std::fmt::Display for StrideType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
StrideType::Stride3 => write!(f, "S3"),
StrideType::Stride4 => write!(f, "S4"),
StrideType::Stride5 => write!(f, "S5"),
}
}
}
pub struct TreeBitMap<
AF: AddressFamily,
M: Meta + MergeUpdate,
NB: NodeBuckets<AF>,
PB: PrefixBuckets<AF, M>,
> {
pub stats: Vec<StrideStats>,
pub store: CustomAllocStorage<AF, M, NB, PB>,
}
impl<
'a,
AF: AddressFamily,
M: Meta + MergeUpdate,
NB: NodeBuckets<AF>,
PB: PrefixBuckets<AF, M>,
> TreeBitMap<AF, M, NB, PB>
{
pub fn new(
) -> Result<TreeBitMap<AF, M, NB, PB>, Box<dyn std::error::Error>> {
let mut stride_stats: Vec<StrideStats> = vec![
StrideStats::new(
SizedStride::Stride3,
CustomAllocStorage::<AF, M, NB, PB>::get_strides_len() as u8,
), StrideStats::new(
SizedStride::Stride4,
CustomAllocStorage::<AF, M, NB, PB>::get_strides_len() as u8,
), StrideStats::new(
SizedStride::Stride5,
CustomAllocStorage::<AF, M, NB, PB>::get_strides_len() as u8,
), ];
let root_node: SizedStrideNode<AF>;
let guard = &epoch::pin();
match CustomAllocStorage::<AF, M, NB, PB>::get_first_stride_size() {
3 => {
root_node = SizedStrideNode::Stride3(TreeBitMapNode {
ptrbitarr: AtomicStride2(AtomicU8::new(0)),
pfxbitarr: AtomicStride3(AtomicU16::new(0)),
_af: PhantomData,
});
stride_stats[0].inc(0);
}
4 => {
root_node = SizedStrideNode::Stride4(TreeBitMapNode {
ptrbitarr: AtomicStride3(AtomicU16::new(0)),
pfxbitarr: AtomicStride4(AtomicU32::new(0)),
_af: PhantomData,
});
stride_stats[1].inc(0);
}
5 => {
root_node = SizedStrideNode::Stride5(TreeBitMapNode {
ptrbitarr: AtomicStride4(AtomicU32::new(0)),
pfxbitarr: AtomicStride5(AtomicU64::new(0)),
_af: PhantomData,
});
stride_stats[2].inc(0);
}
unknown_stride_size => {
panic!(
"unknown stride size {} encountered in STRIDES array",
unknown_stride_size
);
}
};
Ok(TreeBitMap {
stats: stride_stats,
store: CustomAllocStorage::<AF, M, NB, PB>::init(root_node, guard)?,
})
}
pub fn insert(
&self,
pfx: InternalPrefixRecord<AF, M>,
) -> Result<u32, PrefixStoreError> {
let guard = &epoch::pin();
if pfx.len == 0 {
let retry_count = self.update_default_route_prefix_meta(pfx.meta, guard)?;
return Ok(retry_count);
}
let mut stride_end: u8 = 0;
let mut cur_i = self.store.get_root_node_id();
let mut level: u8 = 0;
let mut acc_retry_count = 0;
loop {
let stride = self.store.get_stride_sizes()[level as usize];
stride_end += stride;
let nibble_len = if pfx.len < stride_end {
stride + pfx.len - stride_end
} else {
stride
};
let nibble =
AF::get_nibble(pfx.net, stride_end - stride, nibble_len);
let is_last_stride = pfx.len <= stride_end;
let stride_start = stride_end - stride;
let back_off = crossbeam_utils::Backoff::new();
let node_result = insert_match![
self;
guard;
nibble_len;
nibble;
is_last_stride;
pfx;
stride_start; stride;
cur_i;
level;
back_off;
acc_retry_count;
Stride3; 0,
Stride4; 1,
Stride5; 2
];
match node_result {
Ok((next_id, retry_count)) => {
cur_i = next_id;
level += 1;
acc_retry_count += retry_count;
}
Err(err) => {
if log_enabled!(log::Level::Error) {
error!("{} failing to store (intermediate) node {}. Giving up this node. This shouldn't happen!",
std::thread::current().name().unwrap(),
cur_i,
);
error!("{} {}", std::thread::current().name().unwrap(), err);
}
}
}
};
}
pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
self.store.get_root_node_id()
}
fn update_default_route_prefix_meta(
&self,
new_meta: M,
guard: &epoch::Guard,
) -> Result<u32, PrefixStoreError> {
trace!("Updating the default route...");
self.store.upsert_prefix(
InternalPrefixRecord::new_with_meta(AF::zero(), 0, new_meta),
guard
)
}
fn get_all_more_specifics_for_node(
&self,
start_node_id: StrideNodeId<AF>,
found_pfx_vec: &mut Vec<PrefixId<AF>>,
) {
let guard = &epoch::pin();
trace!("start assembling all more specific prefixes here");
trace!(
"{:?}",
self.store.retrieve_node_with_guard(start_node_id, guard)
);
match self.store.retrieve_node_with_guard(start_node_id, guard) {
Some(SizedStrideRef::Stride3(n)) => {
found_pfx_vec.extend(
n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
);
for child_node in n.ptr_iter(start_node_id) {
self.get_all_more_specifics_for_node(
child_node,
found_pfx_vec,
);
}
}
Some(SizedStrideRef::Stride4(n)) => {
found_pfx_vec.extend(
n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
);
for child_node in n.ptr_iter(start_node_id) {
self.get_all_more_specifics_for_node(
child_node,
found_pfx_vec,
);
}
}
Some(SizedStrideRef::Stride5(n)) => {
found_pfx_vec.extend(
n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
);
for child_node in n.ptr_iter(start_node_id) {
self.get_all_more_specifics_for_node(
child_node,
found_pfx_vec,
);
}
}
_ => {
panic!("can't find node {}", start_node_id);
}
}
}
pub(crate) fn get_all_more_specifics_from_nibble<S: Stride>(
&'a self,
current_node: &TreeBitMapNode<AF, S>,
nibble: u32,
nibble_len: u8,
base_prefix: StrideNodeId<AF>,
) -> Option<Vec<PrefixId<AF>>>
where
S: Stride,
{
let (cnvec, mut msvec) = current_node.add_more_specifics_at(
nibble,
nibble_len,
base_prefix,
);
for child_node in cnvec.iter() {
self.get_all_more_specifics_for_node(*child_node, &mut msvec);
}
Some(msvec)
}
}
impl<
AF: AddressFamily,
M: Meta + MergeUpdate,
NB: NodeBuckets<AF>,
PB: PrefixBuckets<AF, M>,
> Default for TreeBitMap<AF, M, NB, PB>
{
fn default() -> Self {
Self::new().unwrap()
}
}
#[cfg(feature = "cli")]
impl<
'a,
AF: AddressFamily,
M: Meta + MergeUpdate,
NB: NodeBuckets<AF>,
PB: PrefixBuckets<AF, M>,
> std::fmt::Display for TreeBitMap<AF, M, NB, PB>
{
fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let total_nodes = self.store.get_nodes_len();
trace!("prefix vec size {}", self.store.get_prefixes_len());
trace!("finished building tree...");
trace!("{:?} nodes created", total_nodes);
trace!(
"size of node: {} bytes",
std::mem::size_of::<SizedStrideNode<u32>>()
);
trace!(
"memory used by nodes: {}kb",
self.store.get_nodes_len()
* std::mem::size_of::<SizedStrideNode<u32>>()
/ 1024
);
trace!(
"stride division {:?}",
self.store
.get_stride_sizes()
.iter()
.map_while(|s| if s > &0 { Some(*s) } else { None })
.collect::<Vec<_>>()
);
for s in &self.stats {
trace!("{:?}", s);
}
trace!(
"level\t[{}|{}] nodes occupied/max nodes percentage_max_nodes_occupied prefixes",
Colour::Blue.paint("nodes"),
Colour::Green.paint("prefixes")
);
let bars = ["▏", "▎", "▍", "▌", "▋", "▊", "▉"];
let mut stride_bits = [0, 0];
const SCALE: u32 = 5500;
trace!(
"stride_sizes {:?}",
self.store
.get_stride_sizes()
.iter()
.map_while(|s| if s > &0 { Some(*s) } else { None })
.enumerate()
.collect::<Vec<(usize, u8)>>()
);
for stride in self
.store
.get_stride_sizes()
.iter()
.map_while(|s| if s > &0 { Some(*s) } else { None })
.enumerate()
{
stride_bits = [stride_bits[1] + 1, stride_bits[1] + stride.1];
let nodes_num = self
.stats
.iter()
.find(|s| s.stride_len == stride.1)
.unwrap()
.created_nodes[stride.0]
.count as u32;
let prefixes_num = self
.stats
.iter()
.find(|s| s.stride_len == stride.1)
.unwrap()
.prefixes_num[stride.0]
.count as u32;
let n = (nodes_num / SCALE) as usize;
let max_pfx = u128::overflowing_pow(2, stride_bits[1] as u32);
print!("{}-{}\t", stride_bits[0], stride_bits[1]);
for _ in 0..n {
print!("{}", Colour::Blue.paint("█"));
}
print!(
"{}",
Colour::Blue.paint(
bars[((nodes_num % SCALE) / (SCALE / 7)) as usize]
) );
print!(
" {}/{} {:.2}%",
nodes_num,
max_pfx.0,
(nodes_num as f64 / max_pfx.0 as f64) * 100.0
);
print!("\n\t");
let n = (prefixes_num / SCALE) as usize;
for _ in 0..n {
print!("{}", Colour::Green.paint("█"));
}
print!(
"{}",
Colour::Green.paint(
bars[((nodes_num % SCALE) / (SCALE / 7)) as usize]
) );
trace!(" {}", prefixes_num);
}
Ok(())
}
}