use crate::error::{Error, Result};
use crate::storage::sstable::bti::encode_bti_trie_key_from_token;
use crate::storage::sstable::bti::parser::FLAG_HAS_HASH_BYTE;
use crate::util::cassandra_murmur3::{
cassandra_murmur3_normalize_token, cassandra_murmur3_x64_128,
};
use std::collections::BTreeMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PartitionPayload {
DataOffset(u64),
RowsOffset(u64),
}
#[derive(Debug, Clone)]
pub struct PartitionTrieEntry {
key: [u8; 9],
hash_byte: u8,
payload: PartitionPayload,
}
#[derive(Debug, Default)]
pub struct PartitionsTrieWriter {
entries: Vec<PartitionTrieEntry>,
first_raw: Option<([u8; 9], Vec<u8>)>,
last_raw: Option<([u8; 9], Vec<u8>)>,
}
impl PartitionsTrieWriter {
pub fn new() -> Self {
Self {
entries: Vec::new(),
first_raw: None,
last_raw: None,
}
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
#[cfg(test)]
pub(crate) fn retained_raw_key_bytes(&self) -> usize {
self.first_raw.as_ref().map_or(0, |(_, k)| k.len())
+ self.last_raw.as_ref().map_or(0, |(_, k)| k.len())
}
pub fn add_partition(&mut self, raw_key_bytes: &[u8], data_offset: u64) {
self.add_partition_with_payload(raw_key_bytes, PartitionPayload::DataOffset(data_offset));
}
pub fn add_partition_with_payload(&mut self, raw_key_bytes: &[u8], payload: PartitionPayload) {
let (h1, h2) = cassandra_murmur3_x64_128(raw_key_bytes);
let key = encode_bti_trie_key_from_token(cassandra_murmur3_normalize_token(h1));
let hash_byte = h2 as u8;
match &self.first_raw {
Some((min_key, _)) if key >= *min_key => {}
_ => self.first_raw = Some((key, raw_key_bytes.to_vec())),
}
match &self.last_raw {
Some((max_key, _)) if key <= *max_key => {}
_ => self.last_raw = Some((key, raw_key_bytes.to_vec())),
}
self.entries.push(PartitionTrieEntry {
key,
hash_byte,
payload,
});
}
pub fn finish(self) -> Result<Vec<u8>> {
if self.entries.is_empty() {
return Ok(Vec::new());
}
let mut entries = self.entries;
entries.sort_by(|a, b| a.key.cmp(&b.key));
for w in entries.windows(2) {
if w[0].key == w[1].key {
return Err(Error::InvalidInput(
"duplicate partition trie key (token collision) in Partitions.db".to_string(),
));
}
}
let first_key = self.first_raw.map(|(_, k)| k).unwrap_or_default();
let last_key = self.last_raw.map(|(_, k)| k).unwrap_or_default();
let key_count = entries.len() as i64;
let mut buf = Vec::new();
let root_offset = emit_partitions_trie(&entries, &mut buf)?;
let first_pos = buf.len() as i64;
write_with_short_length(&mut buf, &first_key)?;
write_with_short_length(&mut buf, &last_key)?;
buf.extend_from_slice(&first_pos.to_be_bytes());
buf.extend_from_slice(&key_count.to_be_bytes());
buf.extend_from_slice(&(root_offset as i64).to_be_bytes());
Ok(buf)
}
}
fn write_with_short_length(buf: &mut Vec<u8>, bytes: &[u8]) -> Result<()> {
if bytes.len() > u16::MAX as usize {
return Err(Error::InvalidInput(format!(
"partition key too long for Partitions.db short-length prefix: {} bytes",
bytes.len()
)));
}
buf.extend_from_slice(&(bytes.len() as u16).to_be_bytes());
buf.extend_from_slice(bytes);
Ok(())
}
#[cfg(test)]
fn filter_hash_byte(raw_key_bytes: &[u8]) -> u8 {
crate::util::cassandra_murmur3::cassandra_partition_filter_hash_lower_bits(raw_key_bytes) as u8
}
const PARTITION_TRIE_KEY_LEN: usize = 9;
struct PendingNode {
transition_byte: u8,
child_offsets: Vec<(u8, usize)>,
}
fn emit_internal(node: &PendingNode, buf: &mut Vec<u8>) -> Result<usize> {
if node.child_offsets.len() == 256 {
write_dense(&node.child_offsets, buf)
} else {
write_sparse(&node.child_offsets, buf)
}
}
fn pop_and_link(stack: &mut Vec<PendingNode>, buf: &mut Vec<u8>) -> Result<()> {
if let Some(node) = stack.pop() {
let off = emit_internal(&node, buf)?;
if let Some(parent) = stack.last_mut() {
parent.child_offsets.push((node.transition_byte, off));
}
}
Ok(())
}
fn emit_partitions_trie(entries: &[PartitionTrieEntry], buf: &mut Vec<u8>) -> Result<usize> {
if entries.is_empty() {
return Err(Error::InvalidInput(
"cannot emit an empty partition trie".to_string(),
));
}
let mut stack: Vec<PendingNode> = Vec::with_capacity(PARTITION_TRIE_KEY_LEN);
let mut prev_key: Option<[u8; PARTITION_TRIE_KEY_LEN]> = None;
for entry in entries {
let key = entry.key;
let lcp = match prev_key {
None => 0,
Some(prev) => {
let mut l = 0usize;
while l < PARTITION_TRIE_KEY_LEN && key[l] == prev[l] {
l += 1;
}
l
}
};
while stack.len() > lcp + 1 {
pop_and_link(&mut stack, buf)?;
}
while stack.len() < PARTITION_TRIE_KEY_LEN {
let depth = stack.len();
let transition_byte = if depth == 0 { 0 } else { key[depth - 1] };
stack.push(PendingNode {
transition_byte,
child_offsets: Vec::new(),
});
}
let leaf_off = write_leaf(entry.hash_byte, entry.payload, buf)?;
if let Some(deepest) = stack.last_mut() {
deepest
.child_offsets
.push((key[PARTITION_TRIE_KEY_LEN - 1], leaf_off));
}
prev_key = Some(key);
}
while stack.len() > 1 {
pop_and_link(&mut stack, buf)?;
}
match stack.pop() {
Some(root) => emit_internal(&root, buf),
None => Err(Error::InvalidInput(
"partition trie produced no root node".to_string(),
)),
}
}
#[cfg(test)]
enum TrieBuildNode {
Leaf {
hash_byte: u8,
payload: PartitionPayload,
},
Internal {
children: BTreeMap<u8, TrieBuildNode>,
},
}
#[cfg(test)]
fn build_trie(entries: &[PartitionTrieEntry]) -> TrieBuildNode {
let mut root = TrieBuildNode::Internal {
children: BTreeMap::new(),
};
for entry in entries {
insert(&mut root, &entry.key, entry.hash_byte, entry.payload);
}
root
}
#[cfg(test)]
fn insert(node: &mut TrieBuildNode, key: &[u8], hash_byte: u8, payload: PartitionPayload) {
match node {
TrieBuildNode::Internal { children } => {
if key.is_empty() {
children
.entry(0)
.or_insert(TrieBuildNode::Leaf { hash_byte, payload });
return;
}
let first = key[0];
let rest = &key[1..];
if rest.is_empty() {
children.insert(first, TrieBuildNode::Leaf { hash_byte, payload });
} else {
let child = children
.entry(first)
.or_insert_with(|| TrieBuildNode::Internal {
children: BTreeMap::new(),
});
insert(child, rest, hash_byte, payload);
}
}
TrieBuildNode::Leaf { .. } => {
}
}
}
#[cfg(test)]
fn serialize_trie(root: &TrieBuildNode) -> Result<Vec<u8>> {
let mut buf = Vec::new();
let root_offset = write_node(root, &mut buf)?;
buf.extend_from_slice(&(root_offset as u64).to_be_bytes());
Ok(buf)
}
#[cfg(test)]
fn write_node(node: &TrieBuildNode, buf: &mut Vec<u8>) -> Result<usize> {
match node {
TrieBuildNode::Leaf { hash_byte, payload } => write_leaf(*hash_byte, *payload, buf),
TrieBuildNode::Internal { children } => {
let mut child_offsets: Vec<(u8, usize)> = Vec::with_capacity(children.len());
for (&byte, child) in children.iter() {
let off = write_node(child, buf)?;
child_offsets.push((byte, off));
}
if child_offsets.len() == 256 {
write_dense(&child_offsets, buf)
} else {
write_sparse(&child_offsets, buf)
}
}
}
}
fn write_leaf(hash_byte: u8, payload: PartitionPayload, buf: &mut Vec<u8>) -> Result<usize> {
let position: i64 = match payload {
PartitionPayload::DataOffset(data_offset) => {
if data_offset > i64::MAX as u64 {
return Err(Error::InvalidInput(format!(
"Data.db offset {data_offset} too large to encode as a signed BTI position"
)));
}
!(data_offset as i64)
}
PartitionPayload::RowsOffset(rows_offset) => {
if rows_offset > i64::MAX as u64 {
return Err(Error::InvalidInput(format!(
"Rows.db offset {rows_offset} too large to encode as a signed BTI position"
)));
}
rows_offset as i64
}
};
let position_bytes = sized_ints_non_zero_size(position);
debug_assert!((1..=8).contains(&position_bytes));
let payload_bits = FLAG_HAS_HASH_BYTE + (position_bytes as u8 - 1);
debug_assert!(payload_bits <= 16);
let offset = buf.len();
let header = payload_bits & 0x0F;
buf.push(header);
buf.push(hash_byte);
write_sized_int_be(buf, position, position_bytes);
Ok(offset)
}
fn write_sparse(child_offsets: &[(u8, usize)], buf: &mut Vec<u8>) -> Result<usize> {
let count = child_offsets.len();
if count == 0 {
return Err(Error::InvalidInput(
"internal BTI trie node has no children".to_string(),
));
}
if count > 255 {
return Err(Error::InvalidInput(format!(
"BTI Sparse node fan-out {count} exceeds 255; Dense node required"
)));
}
let node_offset = buf.len();
let max_delta = child_offsets
.iter()
.map(|(_, child_off)| node_offset - child_off)
.max()
.unwrap_or(0);
let (ordinal, ptr_bytes) = sparse_ordinal_for_delta(max_delta as u64)?;
let header = ordinal << 4;
buf.push(header);
buf.push(count as u8);
for (byte, _) in child_offsets {
buf.push(*byte);
}
for (_, child_off) in child_offsets {
let delta = (node_offset - child_off) as u64;
write_be_unsigned(buf, delta, ptr_bytes);
}
Ok(node_offset)
}
fn write_dense(child_offsets: &[(u8, usize)], buf: &mut Vec<u8>) -> Result<usize> {
debug_assert_eq!(child_offsets.len(), 256, "write_dense expects full fan-out");
if child_offsets.len() != 256 {
return Err(Error::InvalidInput(format!(
"write_dense requires a full 256-child node, got {}",
child_offsets.len()
)));
}
let start_byte = child_offsets[0].0; let node_offset = buf.len();
let max_delta = child_offsets
.iter()
.map(|(_, child_off)| node_offset - child_off)
.max()
.unwrap_or(0);
let (ordinal, ptr_bytes) = dense_ordinal_for_delta(max_delta as u64)?;
let header = ordinal << 4;
buf.push(header);
buf.push(start_byte);
buf.push((child_offsets.len() - 1) as u8);
for (_, child_off) in child_offsets {
let delta = (node_offset - child_off) as u64;
write_be_unsigned(buf, delta, ptr_bytes);
}
Ok(node_offset)
}
fn dense_ordinal_for_delta(max_delta: u64) -> Result<(u8, usize)> {
if max_delta <= 0xFFFF {
Ok((11, 2))
} else if max_delta <= 0xFF_FFFF {
Ok((12, 3))
} else if max_delta <= 0xFFFF_FFFF {
Ok((13, 4))
} else if max_delta <= 0xFF_FFFF_FFFF {
Ok((14, 5))
} else {
Ok((15, 8))
}
}
fn sparse_ordinal_for_delta(max_delta: u64) -> Result<(u8, usize)> {
if max_delta <= 0xFF {
Ok((5, 1))
} else if max_delta <= 0xFFFF {
Ok((7, 2))
} else if max_delta <= 0xFF_FFFF {
Ok((8, 3))
} else if max_delta <= 0xFF_FFFF_FFFF {
Ok((9, 5))
} else {
Err(Error::InvalidInput(format!(
"BTI Partitions.db trie too large: backward delta {max_delta} exceeds 40 bits"
)))
}
}
fn sized_ints_non_zero_size(value: i64) -> usize {
let abs_value = if value < 0 { !value } else { value } as u64;
if abs_value == 0 {
return 1;
}
let significant_bits = 64 - abs_value.leading_zeros() as usize;
(significant_bits + 1).div_ceil(8).clamp(1, 8)
}
fn write_sized_int_be(buf: &mut Vec<u8>, value: i64, bytes: usize) {
let raw = value as u64;
write_be_unsigned(buf, raw, bytes);
}
fn write_be_unsigned(buf: &mut Vec<u8>, value: u64, bytes: usize) {
let all = value.to_be_bytes();
buf.extend_from_slice(&all[8 - bytes..]);
}
#[derive(Debug, Clone)]
pub struct RowIndexBlock {
pub separator_key: Vec<u8>,
pub block_offset: u64,
pub open_marker: Option<(i32, i64)>,
}
#[derive(Debug, Clone)]
struct PartitionRowIndex {
partition_key: Vec<u8>,
data_position: u64,
blocks: Vec<RowIndexBlock>,
partition_deletion: Option<(i32, i64)>,
}
#[derive(Debug, Default)]
pub struct RowsTrieWriter {
partitions: Vec<PartitionRowIndex>,
}
impl RowsTrieWriter {
pub fn new() -> Self {
Self {
partitions: Vec::new(),
}
}
pub fn len(&self) -> usize {
self.partitions.len()
}
pub fn is_empty(&self) -> bool {
self.partitions.is_empty()
}
pub fn add_partition_row_index(
&mut self,
partition_key: &[u8],
data_position: u64,
blocks: Vec<RowIndexBlock>,
partition_deletion: Option<(i32, i64)>,
) {
self.partitions.push(PartitionRowIndex {
partition_key: partition_key.to_vec(),
data_position,
blocks,
partition_deletion,
});
}
pub fn finish(self) -> Result<(Vec<u8>, Vec<u64>)> {
let mut buf = Vec::new();
let mut rows_offsets = Vec::with_capacity(self.partitions.len());
for part in &self.partitions {
if part.blocks.is_empty() {
return Err(Error::InvalidInput(
"Rows.db: wide partition must have at least one row-index block".to_string(),
));
}
for w in part.blocks.windows(2) {
if w[0].separator_key >= w[1].separator_key {
return Err(Error::InvalidInput(
"Rows.db: row-index separators must be strictly ascending".to_string(),
));
}
}
let root = build_row_trie(&part.blocks);
let trie_root = write_row_node(&root, &mut buf)?;
let rows_offset = buf.len();
write_trie_index_entry(
&mut buf,
&part.partition_key,
part.data_position,
trie_root,
part.blocks.len() as u64,
part.partition_deletion,
)?;
rows_offsets.push(rows_offset as u64);
}
Ok((buf, rows_offsets))
}
}
enum RowTrieBuildNode {
Leaf {
block_offset: u64,
open_marker: Option<(i32, i64)>,
},
Internal {
children: BTreeMap<u8, RowTrieBuildNode>,
},
}
fn build_row_trie(blocks: &[RowIndexBlock]) -> RowTrieBuildNode {
let mut root = RowTrieBuildNode::Internal {
children: BTreeMap::new(),
};
for b in blocks {
insert_row(&mut root, &b.separator_key, b.block_offset, b.open_marker);
}
root
}
fn insert_row(
node: &mut RowTrieBuildNode,
key: &[u8],
block_offset: u64,
open_marker: Option<(i32, i64)>,
) {
match node {
RowTrieBuildNode::Internal { children } => {
if key.is_empty() {
children.entry(0).or_insert(RowTrieBuildNode::Leaf {
block_offset,
open_marker,
});
return;
}
let first = key[0];
let rest = &key[1..];
if rest.is_empty() {
children.insert(
first,
RowTrieBuildNode::Leaf {
block_offset,
open_marker,
},
);
} else {
let child = children
.entry(first)
.or_insert_with(|| RowTrieBuildNode::Internal {
children: BTreeMap::new(),
});
insert_row(child, rest, block_offset, open_marker);
}
}
RowTrieBuildNode::Leaf { .. } => {
}
}
}
fn write_row_node(node: &RowTrieBuildNode, buf: &mut Vec<u8>) -> Result<usize> {
match node {
RowTrieBuildNode::Leaf {
block_offset,
open_marker,
} => write_row_leaf(*block_offset, *open_marker, buf),
RowTrieBuildNode::Internal { children } => {
let mut child_offsets: Vec<(u8, usize)> = Vec::with_capacity(children.len());
for (&byte, child) in children.iter() {
let off = write_row_node(child, buf)?;
child_offsets.push((byte, off));
}
if child_offsets.len() == 256 {
write_dense(&child_offsets, buf)
} else {
write_sparse(&child_offsets, buf)
}
}
}
}
fn write_row_leaf(
block_offset: u64,
open_marker: Option<(i32, i64)>,
buf: &mut Vec<u8>,
) -> Result<usize> {
if block_offset > i64::MAX as u64 {
return Err(Error::InvalidInput(format!(
"Rows.db block offset {block_offset} too large to encode as SizedInts"
)));
}
let offset_bytes = sized_ints_non_zero_size(block_offset as i64);
if !(1..=7).contains(&offset_bytes) {
return Err(Error::InvalidInput(format!(
"Rows.db block offset {block_offset} needs {offset_bytes} SizedInts bytes; \
expected 1..=7"
)));
}
let mut payload_bits = offset_bytes as u8;
if open_marker.is_some() {
payload_bits |= crate::storage::sstable::bti::parser::FLAG_OPEN_MARKER;
}
let offset = buf.len();
buf.push(payload_bits & 0x0F);
write_sized_int_be(buf, block_offset as i64, offset_bytes);
if let Some((ldt, mfda)) = open_marker {
write_da_deletion_time(buf, Some((ldt, mfda)));
}
Ok(offset)
}
fn write_trie_index_entry(
buf: &mut Vec<u8>,
partition_key: &[u8],
data_position: u64,
trie_root: usize,
block_count: u64,
partition_deletion: Option<(i32, i64)>,
) -> Result<()> {
let entry_start = buf.len();
let key_length = partition_key.len();
let key_length_u16 = u16::try_from(key_length).map_err(|_| {
Error::InvalidInput(format!(
"Rows.db TrieIndexEntry: partition key length {key_length} exceeds u16"
))
})?;
buf.extend_from_slice(&key_length_u16.to_be_bytes());
buf.extend_from_slice(partition_key);
write_unsigned_vint(buf, data_position);
let base = entry_start + key_length;
let root_delta = trie_root as i64 - base as i64;
write_signed_vint(buf, root_delta);
write_unsigned_vint(buf, block_count);
write_da_deletion_time(buf, partition_deletion);
Ok(())
}
fn write_unsigned_vint(buf: &mut Vec<u8>, value: u64) {
let extra_bytes = if value == 0 {
0
} else {
let significant_bits = 64 - value.leading_zeros() as usize;
let mut n = 0usize;
while n < 8 && (7 - n) + 8 * n < significant_bits {
n += 1;
}
n
};
if extra_bytes == 0 {
buf.push(value as u8);
return;
}
if extra_bytes >= 8 {
buf.push(0xFF);
buf.extend_from_slice(&value.to_be_bytes());
return;
}
let data_bits_first = 7 - extra_bytes;
let leading_ones: u8 = (!0u8) << (8 - extra_bytes);
let total_bytes = extra_bytes + 1;
let mut bytes = value.to_be_bytes().to_vec();
let tail = bytes.split_off(8 - total_bytes);
let first = leading_ones | (tail[0] & ((1u8 << data_bits_first) - 1));
buf.push(first);
buf.extend_from_slice(&tail[1..]);
}
fn write_signed_vint(buf: &mut Vec<u8>, value: i64) {
let zigzag = ((value << 1) ^ (value >> 63)) as u64;
write_unsigned_vint(buf, zigzag);
}
fn write_da_deletion_time(buf: &mut Vec<u8>, deletion: Option<(i32, i64)>) {
match deletion {
None => buf.push(0x80),
Some((local_deletion_time, marked_for_delete_at)) => {
buf.extend_from_slice(&marked_for_delete_at.to_be_bytes());
buf.extend_from_slice(&(local_deletion_time as u32).to_be_bytes());
}
}
}
#[cfg(test)]
#[path = "partitions_writer_tests.rs"]
mod tests;