use std::hash::Hash;
use std::slice;
use crate::codec::SketchBytes;
use crate::codec::SketchSlice;
use crate::codec::assert::ensure_preamble_longs_in_range;
use crate::codec::assert::insufficient_data;
use crate::codec::family::Family;
use crate::common::NumStdDev;
use crate::common::ResizeFactor;
use crate::error::Error;
use crate::error::ErrorKind;
use crate::hash::DEFAULT_UPDATE_SEED;
use crate::hash::check_seed_hash;
use crate::hash::compute_seed_hash;
use crate::thetacommon::EntrySketch;
use crate::thetacommon::KeySketch;
use crate::thetacommon::binomial_bounds;
use crate::thetacommon::constants::DEFAULT_LG_K;
use crate::thetacommon::constants::FLAGS_IS_COMPACT;
use crate::thetacommon::constants::FLAGS_IS_EMPTY;
use crate::thetacommon::constants::FLAGS_IS_ORDERED;
use crate::thetacommon::constants::FLAGS_IS_READ_ONLY;
use crate::thetacommon::constants::MAX_THETA;
use crate::thetacommon::hash_table::SketchHashTableIter;
use crate::thetacommon::sketch_state::CompactSketchState;
use crate::thetacommon::sketch_state::ThetaFamilySketchMetadata;
use crate::tuple::hash_table::TupleEntry;
use crate::tuple::hash_table::TupleHashTable;
use crate::tuple::policy::SummaryPolicy;
use crate::tuple::policy::SummaryUpdatePolicy;
use crate::tuple::serialization::SERIAL_VERSION;
use crate::tuple::serialization::SERIAL_VERSION_LEGACY;
use crate::tuple::serialization::SKETCH_TYPE;
use crate::tuple::serialization::SKETCH_TYPE_LEGACY;
use crate::tuple::serialization::TupleSummaryValue;
#[derive(Debug)]
pub struct TupleSketchView<'a, S>(TupleSketchViewState<'a, S>);
#[derive(Debug)]
enum TupleSketchViewState<'a, S> {
Mutable {
table: &'a TupleHashTable<S>,
is_empty: bool,
},
Compact(&'a CompactTupleSketch<S>),
}
enum TupleSketchIter<'a, S> {
Mutable(SketchHashTableIter<'a, TupleEntry<S>>),
Compact(slice::Iter<'a, TupleEntry<S>>),
}
impl<'a, S> Iterator for TupleSketchIter<'a, S> {
type Item = &'a TupleEntry<S>;
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Mutable(iter) => iter.next(),
Self::Compact(iter) => iter.next(),
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
match self {
Self::Mutable(iter) => iter.size_hint(),
Self::Compact(iter) => iter.size_hint(),
}
}
}
impl<S> Clone for TupleSketchView<'_, S> {
fn clone(&self) -> Self {
*self
}
}
impl<S> Copy for TupleSketchView<'_, S> {}
impl<S> Clone for TupleSketchViewState<'_, S> {
fn clone(&self) -> Self {
*self
}
}
impl<S> Copy for TupleSketchViewState<'_, S> {}
impl<'a, S> TupleSketchView<'a, S> {
pub fn seed_hash(&self) -> u16 {
match self.0 {
TupleSketchViewState::Mutable { table, .. } => table.seed_hash(),
TupleSketchViewState::Compact(sketch) => sketch.seed_hash(),
}
}
pub fn theta64(&self) -> u64 {
match self.0 {
TupleSketchViewState::Mutable { table, is_empty } => {
if is_empty {
MAX_THETA
} else {
table.retention_theta()
}
}
TupleSketchViewState::Compact(sketch) => sketch.theta64(),
}
}
pub fn is_empty(&self) -> bool {
match self.0 {
TupleSketchViewState::Mutable { is_empty, .. } => is_empty,
TupleSketchViewState::Compact(sketch) => sketch.is_empty(),
}
}
pub fn is_ordered(&self) -> bool {
match self.0 {
TupleSketchViewState::Mutable { .. } => false,
TupleSketchViewState::Compact(sketch) => sketch.is_ordered(),
}
}
pub fn iter(self) -> impl Iterator<Item = &'a TupleEntry<S>> + 'a {
match self.0 {
TupleSketchViewState::Mutable { table, .. } => {
TupleSketchIter::Mutable(table.iter_entries())
}
TupleSketchViewState::Compact(sketch) => {
TupleSketchIter::Compact(sketch.compact_state.retained_entries().iter())
}
}
}
pub fn num_retained(&self) -> usize {
match self.0 {
TupleSketchViewState::Mutable { table, .. } => table.num_retained(),
TupleSketchViewState::Compact(sketch) => sketch.num_retained(),
}
}
}
impl<S> KeySketch for TupleSketchView<'_, S> {
fn metadata(self) -> ThetaFamilySketchMetadata {
if self.is_empty() {
ThetaFamilySketchMetadata::Empty {
seed_hash: self.seed_hash(),
}
} else {
ThetaFamilySketchMetadata::NonEmpty {
seed_hash: self.seed_hash(),
theta: self.theta64(),
ordered: self.is_ordered(),
num_retained: self.num_retained(),
}
}
}
fn hashes(self) -> impl Iterator<Item = u64> {
self.iter().map(TupleEntry::hash)
}
}
impl<'a, S> EntrySketch for TupleSketchView<'a, S>
where
S: Clone + 'a,
{
type Entry = TupleEntry<S>;
fn entries(self) -> impl Iterator<Item = Self::Entry> {
self.iter().cloned()
}
}
impl<'a, P> From<&'a TupleSketch<P>> for TupleSketchView<'a, P::Summary>
where
P: SummaryPolicy,
{
fn from(sketch: &'a TupleSketch<P>) -> Self {
Self(TupleSketchViewState::Mutable {
table: &sketch.table,
is_empty: sketch.is_empty,
})
}
}
impl<'a, S> From<&'a CompactTupleSketch<S>> for TupleSketchView<'a, S> {
fn from(sketch: &'a CompactTupleSketch<S>) -> Self {
Self(TupleSketchViewState::Compact(sketch))
}
}
#[derive(Debug)]
pub struct TupleSketch<P>
where
P: SummaryPolicy,
{
table: TupleHashTable<P::Summary>,
is_empty: bool,
policy: P,
}
impl<P> TupleSketch<P>
where
P: SummaryPolicy,
{
pub fn as_view(&self) -> TupleSketchView<'_, P::Summary> {
self.into()
}
pub fn update<U>(&mut self, key: impl Hash, value: U)
where
P: SummaryUpdatePolicy<U>,
{
self.is_empty = false;
let policy = &self.policy;
self.table.try_insert(key, |existing| match existing {
Some(summary) => {
policy.update(summary, value);
None
}
None => {
let mut summary = policy.create();
policy.update(&mut summary, value);
Some(summary)
}
});
}
pub fn estimate(&self) -> f64 {
if self.is_empty() {
return 0.0;
}
let num_retained = self.table.num_retained() as f64;
let theta = self.theta64() as f64 / MAX_THETA as f64;
num_retained / theta
}
pub fn theta(&self) -> f64 {
self.theta64() as f64 / MAX_THETA as f64
}
pub fn theta64(&self) -> u64 {
if self.is_empty {
MAX_THETA
} else {
self.table.retention_theta()
}
}
pub fn seed_hash(&self) -> u16 {
self.table.seed_hash()
}
pub fn is_empty(&self) -> bool {
self.is_empty
}
pub fn is_estimation_mode(&self) -> bool {
!self.is_empty && self.table.retention_theta() < MAX_THETA
}
pub fn num_retained(&self) -> usize {
self.table.num_retained()
}
pub fn lg_k(&self) -> u8 {
self.table.lg_nom_size()
}
pub fn trim(&mut self) {
self.table.trim();
}
pub fn reset(&mut self) {
self.table.reset();
self.is_empty = true;
}
pub fn iter(&self) -> impl Iterator<Item = &TupleEntry<P::Summary>> + '_ {
self.table.iter()
}
pub fn lower_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::lower_bound(self.num_retained() as u64, self.theta(), num_std_dev)
.expect("theta should always be valid")
}
pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::upper_bound(
self.num_retained() as u64,
self.theta(),
num_std_dev,
self.is_empty(),
)
.expect("theta should always be valid")
}
pub fn estimated_size(&self) -> usize {
size_of::<Self>() + self.table.estimated_size()
}
}
impl<P> TupleSketch<P>
where
P: SummaryPolicy,
P::Summary: Clone,
{
pub fn compact(&self, ordered: bool) -> CompactTupleSketch<P::Summary> {
let compact_state = if self.is_empty() {
debug_assert_eq!(self.num_retained(), 0);
CompactSketchState::empty(self.seed_hash())
} else {
self.table.to_non_empty_compact_state(ordered)
};
CompactTupleSketch::from_compact_state(compact_state)
}
}
#[derive(Clone, Debug)]
pub struct CompactTupleSketch<S> {
compact_state: CompactSketchState<TupleEntry<S>>,
}
impl<S> CompactTupleSketch<S> {
pub(super) fn from_compact_state(compact_state: CompactSketchState<TupleEntry<S>>) -> Self {
Self { compact_state }
}
pub fn as_view(&self) -> TupleSketchView<'_, S> {
self.into()
}
pub fn estimate(&self) -> f64 {
if self.is_empty() {
return 0.0;
}
let num_retained = self.num_retained() as f64;
if self.theta64() == MAX_THETA {
return num_retained;
}
let theta = self.theta();
num_retained / theta
}
pub fn theta(&self) -> f64 {
self.theta64() as f64 / MAX_THETA as f64
}
pub fn theta64(&self) -> u64 {
self.compact_state.theta()
}
pub fn is_empty(&self) -> bool {
self.compact_state.is_empty()
}
pub fn is_estimation_mode(&self) -> bool {
self.compact_state.is_estimation_mode()
}
pub fn num_retained(&self) -> usize {
self.retained_entries().len()
}
pub fn is_ordered(&self) -> bool {
self.compact_state.is_ordered()
}
pub fn seed_hash(&self) -> u16 {
self.compact_state.seed_hash()
}
pub fn iter(&self) -> impl Iterator<Item = &TupleEntry<S>> + '_ {
self.retained_entries().iter()
}
fn retained_entries(&self) -> &[TupleEntry<S>] {
self.compact_state.retained_entries()
}
pub fn lower_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::lower_bound(self.num_retained() as u64, self.theta(), num_std_dev)
.expect("compact theta should always be valid")
}
pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::upper_bound(
self.num_retained() as u64,
self.theta(),
num_std_dev,
self.is_empty(),
)
.expect("compact theta should always be valid")
}
pub fn estimated_size(&self) -> usize {
size_of::<Self>()
+ self.compact_state.retained_entries_capacity() * size_of::<TupleEntry<S>>()
}
fn preamble_longs(&self) -> u8 {
if self.is_estimation_mode() {
3
} else if self.is_empty() || self.num_retained() == 1 {
1
} else {
2
}
}
pub fn serialize(&self) -> Vec<u8>
where
S: TupleSummaryValue,
{
let retained_entries = self.retained_entries();
let pre_longs = self.preamble_longs();
let entries_size: usize = retained_entries
.iter()
.map(|entry| 8 + entry.summary().serialize_size())
.sum();
let mut bytes = SketchBytes::with_capacity(8 * pre_longs as usize + entries_size);
bytes.write_u8(pre_longs);
bytes.write_u8(SERIAL_VERSION);
bytes.write_u8(Family::TUPLE.id);
bytes.write_u8(SKETCH_TYPE);
bytes.write_u8(0);
let mut flags = FLAGS_IS_READ_ONLY | FLAGS_IS_COMPACT;
if self.is_empty() {
flags |= FLAGS_IS_EMPTY;
}
if self.is_ordered() {
flags |= FLAGS_IS_ORDERED;
}
bytes.write_u8(flags);
bytes.write_u16_le(self.seed_hash());
if pre_longs > 1 {
bytes.write_u32_le(retained_entries.len() as u32);
bytes.write_u32_le(0); }
if self.is_estimation_mode() {
bytes.write_u64_le(self.theta64());
}
for entry in retained_entries {
bytes.write_u64_le(entry.hash());
entry.summary().serialize_value(&mut bytes);
}
bytes.into_bytes()
}
pub fn deserialize(bytes: &[u8]) -> Result<Self, Error>
where
S: TupleSummaryValue,
{
Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED)
}
pub fn deserialize_with_seed(bytes: &[u8], seed: u64) -> Result<Self, Error>
where
S: TupleSummaryValue,
{
let expected_seed_hash = compute_seed_hash(seed, ErrorKind::InvalidData)?;
let mut cursor = SketchSlice::new(bytes);
let pre_longs = cursor
.read_u8()
.map_err(insufficient_data("preamble_longs"))?;
let ser_ver = cursor
.read_u8()
.map_err(insufficient_data("serial_version"))?;
let family_id = cursor.read_u8().map_err(insufficient_data("family_id"))?;
let sketch_type = cursor.read_u8().map_err(insufficient_data("sketch_type"))?;
cursor.read_u8().map_err(insufficient_data("<unused>"))?;
let flags = cursor.read_u8().map_err(insufficient_data("flags"))?;
let seed_hash = cursor
.read_u16_le()
.map_err(insufficient_data("seed_hash"))?;
Family::TUPLE.validate_id(family_id)?;
ensure_preamble_longs_in_range(
Family::TUPLE.min_pre_longs..=Family::TUPLE.max_pre_longs,
pre_longs,
)?;
if ser_ver != SERIAL_VERSION && ser_ver != SERIAL_VERSION_LEGACY {
return Err(Error::deserial(format!(
"unsupported serial version: expected {} or {}, got {ser_ver}",
SERIAL_VERSION, SERIAL_VERSION_LEGACY,
)));
}
if sketch_type != SKETCH_TYPE && sketch_type != SKETCH_TYPE_LEGACY {
return Err(Error::deserial(format!(
"unsupported sketch type: expected {} or {}, got {sketch_type}",
SKETCH_TYPE, SKETCH_TYPE_LEGACY,
)));
}
let empty = (flags & FLAGS_IS_EMPTY) != 0;
let ordered = (flags & FLAGS_IS_ORDERED) != 0;
if empty {
return Ok(Self::from_compact_state(CompactSketchState::empty(
seed_hash,
)));
}
check_seed_hash(
expected_seed_hash,
seed_hash,
"deserialized CompactTupleSketch",
ErrorKind::InvalidData,
)?;
let mut theta = MAX_THETA;
let num_entries = if pre_longs == 1 {
1
} else {
let n = cursor
.read_u32_le()
.map_err(insufficient_data("num_entries"))? as usize;
cursor
.read_u32_le()
.map_err(insufficient_data("<unused_u32>"))?;
if pre_longs > 2 {
let value = cursor.read_u64_le().map_err(insufficient_data("theta"))?;
if !(1..=MAX_THETA).contains(&value) {
return Err(Error::deserial(format!(
"corrupted: theta must be in [1, {MAX_THETA}], got {value}"
)));
}
theta = value;
}
n
};
let required_hash_bytes = num_entries
.checked_mul(size_of::<u64>())
.ok_or_else(|| Error::deserial("Tuple entry payload length overflows"))?;
if required_hash_bytes > cursor.remaining().len() {
return Err(Error::insufficient_data(format!(
"Tuple entry hashes require at least {required_hash_bytes} bytes, got {}",
cursor.remaining().len()
)));
}
let mut retained_entries = Vec::with_capacity(num_entries);
for _ in 0..num_entries {
let hash = cursor
.read_u64_le()
.map_err(insufficient_data("entry_hash"))?;
if hash == 0 || hash >= theta {
return Err(Error::deserial("corrupted: invalid retained hash value"));
}
let summary = S::deserialize_value(&mut cursor)?;
retained_entries.push(TupleEntry::new(hash, summary));
}
Ok(Self::from_compact_state(CompactSketchState::non_empty(
retained_entries,
theta,
seed_hash,
ordered,
)))
}
}
#[derive(Debug)]
pub struct TupleSketchBuilder<P>
where
P: SummaryPolicy,
{
lg_k: u8,
resize_factor: ResizeFactor,
sampling_probability: f32,
seed: u64,
policy: P,
}
impl<P> TupleSketchBuilder<P>
where
P: SummaryPolicy,
{
pub fn new(policy: P) -> Self {
Self {
lg_k: DEFAULT_LG_K,
resize_factor: ResizeFactor::X8,
sampling_probability: 1.0,
seed: DEFAULT_UPDATE_SEED,
policy,
}
}
pub fn lg_k(mut self, lg_k: u8) -> Self {
self.lg_k = lg_k;
self
}
pub fn resize_factor(mut self, factor: ResizeFactor) -> Self {
self.resize_factor = factor;
self
}
pub fn sampling_probability(mut self, probability: f32) -> Self {
self.sampling_probability = probability;
self
}
pub fn seed(mut self, seed: u64) -> Self {
self.seed = seed;
self
}
pub fn build(self) -> Result<TupleSketch<P>, Error> {
Ok(TupleSketch {
table: TupleHashTable::new(
self.lg_k,
self.resize_factor,
self.sampling_probability,
self.seed,
)?,
is_empty: true,
policy: self.policy,
})
}
}