mod encoding_stats;
mod header;
mod marshal_type;
mod serialization_header;
pub use header::parse_nb_format_header;
use super::statistics::*;
use crate::error::{Error, Result};
use crate::parser::repair_metadata::{parse_statistics_toc, StatisticsToc};
use crate::storage::sstable::version_gate::VersionGates;
use encoding_stats::parse_minimal_encoding_stats;
type EncodingStatsResult = (
i64,
i64,
Option<i64>,
Vec<super::header::ColumnInfo>,
Vec<super::header::ColumnInfo>,
Vec<super::header::ColumnInfo>,
);
type SerializationHeaderResult = (Vec<String>, Vec<String>, Vec<super::header::ColumnInfo>);
#[allow(clippy::type_complexity)]
pub fn parse_nb_format_statistics_data(
input: &[u8],
header: &StatisticsHeader,
full_input: &[u8],
gates: Option<&VersionGates>,
) -> Result<(
RowStatistics,
TimestampStatistics,
Vec<super::header::ColumnInfo>,
Vec<super::header::ColumnInfo>,
Vec<super::header::ColumnInfo>,
)> {
let toc = parse_statistics_toc(full_input);
parse_nb_format_statistics_data_with_toc(input, header, full_input, &toc, gates)
}
#[allow(clippy::type_complexity)]
pub(crate) fn parse_nb_format_statistics_data_with_toc(
input: &[u8],
header: &StatisticsHeader,
full_input: &[u8],
toc: &StatisticsToc,
gates: Option<&VersionGates>,
) -> Result<(
RowStatistics,
TimestampStatistics,
Vec<super::header::ColumnInfo>,
Vec<super::header::ColumnInfo>,
Vec<super::header::ColumnInfo>,
)> {
let header_offset = toc.header_offset();
let result = parse_minimal_encoding_stats(input, full_input, header_offset, gates);
match result {
Ok((
_,
(
min_timestamp,
min_deletion_time,
min_ttl,
partition_columns,
clustering_columns,
regular_columns,
),
)) => {
let (total_rows, partition_count) =
match crate::parser::repair_metadata::read_table_counts_with_toc(
full_input, toc, gates,
) {
Ok(counts) => (counts.total_rows.unwrap_or(0), counts.partition_count),
Err(e) => {
tracing::debug!(
"Best-effort authoritative row/partition-count decode failed; \
leaving RowStatistics counts at 0: {:?}",
e
);
(0, 0)
}
};
let avg_rows_per_partition = if partition_count > 0 && total_rows > 0 {
total_rows as f64 / partition_count as f64
} else {
0.0
};
let row_stats = RowStatistics {
total_rows,
live_rows: 0,
tombstone_count: 0,
partition_count,
avg_rows_per_partition,
row_size_histogram: vec![],
};
let timestamp_stats = TimestampStatistics {
min_timestamp,
max_timestamp: None,
min_deletion_time,
max_deletion_time: i64::MAX,
min_ttl,
max_ttl: None,
rows_with_ttl: None,
};
Ok((
row_stats,
timestamp_stats,
partition_columns,
clustering_columns,
regular_columns,
))
}
Err(e) => {
tracing::debug!(
"Failed to parse minimal EncodingStats from Statistics.db: {:?}",
e
);
Err(Error::UnsupportedFormat(format!(
"Failed to parse minimal nb-format Statistics.db EncodingStats: {:?}. \
This is required for delta-coded timestamp decoding. \
Header checksum: 0x{:08x}, data_length: {}",
e, header.checksum, header.data_length
)))
}
}
}
pub fn parse_enhanced_statistics_file<'a>(
input: &'a [u8],
gates: Option<&VersionGates>,
) -> nom::IResult<&'a [u8], SSTableStatistics> {
let (remaining, header) = parse_nb_format_header(input)?;
let toc = parse_statistics_toc(input);
let result = parse_nb_format_statistics_data_with_toc(remaining, &header, input, &toc, gates);
match result {
Ok((row_stats, mut timestamp_stats, partition_columns, clustering_columns, columns)) => {
tracing::debug!(
"Successfully parsed Statistics.db serialization header: {} partition keys, {} clustering keys, {} regular columns",
partition_columns.len(),
clustering_columns.len(),
columns.len()
);
let tombstone_drop_times =
match crate::parser::repair_metadata::parse_stats_extras_with_toc(
input, &toc, gates,
) {
Ok(extras) => {
timestamp_stats.max_deletion_time = extras.max_local_deletion_time;
if let Some(max_ts) = extras.max_timestamp {
timestamp_stats.max_timestamp = Some(max_ts);
}
if extras.max_ttl.is_some() {
timestamp_stats.max_ttl = extras.max_ttl;
}
extras.tombstone_drop_times
}
Err(e) => {
tracing::debug!(
"Best-effort STATS-extras decode failed; keeping placeholders: {:?}",
e
);
vec![]
}
};
let statistics = SSTableStatistics {
header,
row_stats,
timestamp_stats,
column_stats: vec![],
table_stats: None,
partition_stats: None,
compression_stats: None,
metadata: std::collections::HashMap::new(),
serialization_header_columns: columns,
serialization_header_partition_keys: partition_columns,
serialization_header_clustering_keys: clustering_columns,
tombstone_drop_times,
};
Ok((remaining, statistics))
}
Err(e) => {
tracing::warn!("Failed to parse nb-format Statistics.db: {}", e);
Err(nom::Err::Error(nom::error::Error::new(
input,
nom::error::ErrorKind::Verify,
)))
}
}
}
pub fn parse_statistics_with_fallback<'a>(
input: &'a [u8],
gates: Option<&VersionGates>,
) -> nom::IResult<&'a [u8], SSTableStatistics> {
parse_enhanced_statistics_file(input, gates)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_statistics_data_extraction_with_invalid_data() {
let header = StatisticsHeader {
version: 4,
statistics_kind: 0x2629_1b05,
data_length: 44,
metadata1: 1,
metadata2: 101,
metadata3: 2,
checksum: 0x14d4,
table_id: None,
};
let dummy_data = vec![0xFF; 10]; let result = parse_nb_format_statistics_data(&dummy_data, &header, &dummy_data, None);
assert!(result.is_err());
}
#[test]
fn test_enhanced_statistics_file_with_incomplete_data() {
let test_data = vec![
0x00, 0x00, 0x00, 0x04, 0x26, 0x29, 0x1b, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x14,
0xd4, ];
let result = parse_enhanced_statistics_file(&test_data, None);
assert!(result.is_err());
}
#[test]
fn test_parser_fallback_with_incomplete_data() {
let test_data = vec![
0x00, 0x00, 0x00, 0x04, 0x26, 0x29, 0x1b, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x14, 0xd4, ];
let result = parse_statistics_with_fallback(&test_data, None);
assert!(result.is_err());
}
#[test]
fn test_invalid_data_returns_error() {
let invalid_data = vec![0xFF; 10];
let result = parse_statistics_with_fallback(&invalid_data, None);
assert!(result.is_err(), "Invalid data should fail to parse");
}
#[cfg(test)]
fn real_nb_statistics_db() -> Option<Vec<u8>> {
let root = std::env::var("CQLITE_DATASETS_ROOT").ok()?;
let path = std::path::PathBuf::from(root).join(
"sstables/test_timeseries/sensor_data-6c698230a25111f0a3fef1a551383fb9/\
nb-1-big-Statistics.db",
);
if !path.exists() {
eprintln!("dataset Statistics.db absent, skipping: {path:?}");
return None;
}
std::fs::read(&path).ok()
}
#[test]
fn inner_parser_leaves_unavailable_timestamp_fields_none() {
let Some(bytes) = real_nb_statistics_db() else {
return;
};
let (remaining, header) =
parse_nb_format_header(&bytes).expect("nb header must parse on a real fixture");
let (_row, ts_stats, _pk, _ck, _cols) =
parse_nb_format_statistics_data(remaining, &header, &bytes, None)
.expect("inner EncodingStats parse must succeed on a real fixture");
assert_ne!(
ts_stats.min_timestamp,
i64::MIN,
"fixture must carry a real min_timestamp"
);
assert_eq!(
ts_stats.max_timestamp, None,
"inner parser must leave max_timestamp None (not fabricated as min); issue #1653"
);
assert_eq!(
ts_stats.max_ttl, None,
"inner parser must leave max_ttl None (not fabricated as min_ttl); issue #1653"
);
assert_eq!(
ts_stats.rows_with_ttl, None,
"inner parser must leave rows_with_ttl None (not fabricated as 0); issue #1653"
);
}
#[test]
fn enhanced_parse_leaves_unparsed_stat_blocks_none() {
let Some(bytes) = real_nb_statistics_db() else {
return;
};
let (_remaining, stats) = parse_enhanced_statistics_file(&bytes, None)
.expect("enhanced parse must succeed on a real fixture");
assert!(
stats.table_stats.is_none(),
"enhanced nb parser must not fabricate table_stats; issue #1653"
);
assert!(
stats.partition_stats.is_none(),
"enhanced nb parser must not fabricate partition_stats; issue #1653"
);
assert!(
stats.compression_stats.is_none(),
"enhanced nb parser must not fabricate compression_stats; issue #1653"
);
}
#[test]
fn enhanced_parse_recovers_real_max_timestamp_as_some() {
let Some(bytes) = real_nb_statistics_db() else {
return;
};
let (_remaining, stats) = parse_enhanced_statistics_file(&bytes, None)
.expect("enhanced parse must succeed on a real fixture");
let min = stats.timestamp_stats.min_timestamp;
let max = stats
.timestamp_stats
.max_timestamp
.expect("a real time-series nb SSTable must expose an authoritative maxTimestamp");
assert!(
max >= min,
"recovered max_timestamp ({max}) must be a real max >= min ({min})"
);
}
}