use std::hash::Hash;
use std::slice;
use crate::codec::SketchBytes;
use crate::codec::SketchSlice;
use crate::codec::assert::ensure_preamble_longs_in_range;
use crate::codec::assert::insufficient_data;
use crate::codec::family::Family;
use crate::common::NumStdDev;
use crate::common::ResizeFactor;
use crate::error::Error;
use crate::error::ErrorKind;
use crate::hash::DEFAULT_UPDATE_SEED;
use crate::hash::check_seed_hash;
use crate::hash::compute_seed_hash;
use crate::thetacommon::EntrySketch;
use crate::thetacommon::KeySketch;
use crate::thetacommon::SketchScalars;
use crate::thetacommon::binomial_bounds;
use crate::thetacommon::constants::DEFAULT_LG_K;
use crate::thetacommon::constants::FLAGS_IS_COMPACT;
use crate::thetacommon::constants::FLAGS_IS_EMPTY;
use crate::thetacommon::constants::FLAGS_IS_ORDERED;
use crate::thetacommon::constants::FLAGS_IS_READ_ONLY;
use crate::thetacommon::constants::MAX_LG_K;
use crate::thetacommon::constants::MAX_THETA;
use crate::thetacommon::constants::MIN_LG_K;
use crate::thetacommon::hash_table::SketchHashTableIter;
use crate::tuple::hash_table::TupleEntry;
use crate::tuple::hash_table::TupleHashTable;
use crate::tuple::policy::SummaryPolicy;
use crate::tuple::policy::SummaryUpdatePolicy;
use crate::tuple::serialization::SERIAL_VERSION;
use crate::tuple::serialization::SERIAL_VERSION_LEGACY;
use crate::tuple::serialization::SKETCH_TYPE;
use crate::tuple::serialization::SKETCH_TYPE_LEGACY;
use crate::tuple::serialization::TupleSummaryValue;
#[derive(Debug)]
pub struct TupleSketchView<'a, S>(TupleSketchViewState<'a, S>);
#[derive(Debug)]
enum TupleSketchViewState<'a, S> {
Mutable(&'a TupleHashTable<S>),
Compact(&'a CompactTupleSketch<S>),
}
enum TupleSketchIter<'a, S> {
Mutable(SketchHashTableIter<'a, TupleEntry<S>>),
Compact(slice::Iter<'a, TupleEntry<S>>),
}
impl<'a, S> Iterator for TupleSketchIter<'a, S> {
type Item = (u64, &'a S);
fn next(&mut self) -> Option<Self::Item> {
match self {
Self::Mutable(iter) => iter.next().map(|entry| (entry.hash(), entry.summary())),
Self::Compact(iter) => iter.next().map(|entry| (entry.hash(), entry.summary())),
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
match self {
Self::Mutable(iter) => iter.size_hint(),
Self::Compact(iter) => iter.size_hint(),
}
}
}
impl<S> Clone for TupleSketchView<'_, S> {
fn clone(&self) -> Self {
*self
}
}
impl<S> Copy for TupleSketchView<'_, S> {}
impl<S> Clone for TupleSketchViewState<'_, S> {
fn clone(&self) -> Self {
*self
}
}
impl<S> Copy for TupleSketchViewState<'_, S> {}
impl<'a, S> TupleSketchView<'a, S> {
pub fn seed_hash(&self) -> u16 {
match self.0 {
TupleSketchViewState::Mutable(table) => table.seed_hash(),
TupleSketchViewState::Compact(sketch) => sketch.seed_hash(),
}
}
pub fn theta64(&self) -> u64 {
match self.0 {
TupleSketchViewState::Mutable(table) => table.theta(),
TupleSketchViewState::Compact(sketch) => sketch.theta64(),
}
}
pub fn is_empty(&self) -> bool {
match self.0 {
TupleSketchViewState::Mutable(table) => table.is_empty(),
TupleSketchViewState::Compact(sketch) => sketch.is_empty(),
}
}
pub fn is_ordered(&self) -> bool {
match self.0 {
TupleSketchViewState::Mutable(_) => false,
TupleSketchViewState::Compact(sketch) => sketch.is_ordered(),
}
}
pub fn iter(self) -> impl Iterator<Item = (u64, &'a S)> + 'a {
match self.0 {
TupleSketchViewState::Mutable(table) => TupleSketchIter::Mutable(table.iter_entries()),
TupleSketchViewState::Compact(sketch) => {
TupleSketchIter::Compact(sketch.entries.iter())
}
}
}
pub fn num_retained(&self) -> usize {
match self.0 {
TupleSketchViewState::Mutable(table) => table.num_retained(),
TupleSketchViewState::Compact(sketch) => sketch.num_retained(),
}
}
}
impl<S> KeySketch for TupleSketchView<'_, S> {
fn scalars(self) -> SketchScalars {
SketchScalars {
seed_hash: self.seed_hash(),
theta: self.theta64(),
empty: self.is_empty(),
ordered: self.is_ordered(),
num_retained: self.num_retained(),
}
}
fn hashes(self) -> impl Iterator<Item = u64> {
self.iter().map(|(hash, _)| hash)
}
}
impl<'a, S> EntrySketch for TupleSketchView<'a, S>
where
S: Clone + 'a,
{
type Entry = TupleEntry<S>;
fn entries(self) -> impl Iterator<Item = Self::Entry> {
self.iter()
.map(|(hash, summary)| TupleEntry::new(hash, summary.clone()))
}
}
impl<'a, P> From<&'a TupleSketch<P>> for TupleSketchView<'a, P::Summary>
where
P: SummaryPolicy,
{
fn from(sketch: &'a TupleSketch<P>) -> Self {
Self(TupleSketchViewState::Mutable(&sketch.table))
}
}
impl<'a, S> From<&'a CompactTupleSketch<S>> for TupleSketchView<'a, S> {
fn from(sketch: &'a CompactTupleSketch<S>) -> Self {
Self(TupleSketchViewState::Compact(sketch))
}
}
#[derive(Debug)]
pub struct TupleSketch<P>
where
P: SummaryPolicy,
{
table: TupleHashTable<P::Summary>,
policy: P,
}
impl<P> TupleSketch<P>
where
P: SummaryPolicy,
{
pub fn as_view(&self) -> TupleSketchView<'_, P::Summary> {
self.into()
}
pub fn update<U>(&mut self, key: impl Hash, value: U)
where
P: SummaryUpdatePolicy<U>,
{
let policy = &self.policy;
self.table.try_insert(key, |existing| match existing {
Some(summary) => {
policy.update(summary, value);
None
}
None => {
let mut summary = policy.create();
policy.update(&mut summary, value);
Some(summary)
}
});
}
pub fn estimate(&self) -> f64 {
if self.is_empty() {
return 0.0;
}
let num_retained = self.table.num_retained() as f64;
let theta = self.table.theta() as f64 / MAX_THETA as f64;
num_retained / theta
}
pub fn theta(&self) -> f64 {
self.table.theta() as f64 / MAX_THETA as f64
}
pub fn theta64(&self) -> u64 {
self.table.theta()
}
pub fn seed_hash(&self) -> u16 {
self.table.seed_hash()
}
pub fn is_empty(&self) -> bool {
self.table.is_empty()
}
pub fn is_estimation_mode(&self) -> bool {
self.table.theta() < MAX_THETA
}
pub fn num_retained(&self) -> usize {
self.table.num_retained()
}
pub fn lg_k(&self) -> u8 {
self.table.lg_nom_size()
}
pub fn trim(&mut self) {
self.table.trim();
}
pub fn reset(&mut self) {
self.table.reset();
}
pub fn iter(&self) -> impl Iterator<Item = (u64, &P::Summary)> + '_ {
self.table.iter()
}
pub fn lower_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::lower_bound(self.num_retained() as u64, self.theta(), num_std_dev)
.expect("theta should always be valid")
}
pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::upper_bound(
self.num_retained() as u64,
self.theta(),
num_std_dev,
self.is_empty(),
)
.expect("theta should always be valid")
}
pub fn estimated_size(&self) -> usize {
size_of::<Self>() + self.table.estimated_size()
}
}
impl<P> TupleSketch<P>
where
P: SummaryPolicy,
P::Summary: Clone,
{
pub fn compact(&self, ordered: bool) -> CompactTupleSketch<P::Summary> {
let parts = self.table.to_compact_parts(ordered);
CompactTupleSketch::from_parts(
parts.entries,
parts.theta,
parts.seed_hash,
parts.ordered,
parts.empty,
)
}
}
#[derive(Clone, Debug)]
pub struct CompactTupleSketch<S> {
entries: Vec<TupleEntry<S>>,
theta: u64,
seed_hash: u16,
ordered: bool,
empty: bool,
}
impl<S> CompactTupleSketch<S> {
pub(super) fn from_parts(
entries: Vec<TupleEntry<S>>,
theta: u64,
seed_hash: u16,
ordered: bool,
empty: bool,
) -> Self {
Self {
entries,
theta,
seed_hash,
ordered,
empty,
}
}
pub fn as_view(&self) -> TupleSketchView<'_, S> {
self.into()
}
pub fn estimate(&self) -> f64 {
if self.is_empty() {
return 0.0;
}
let num_retained = self.num_retained() as f64;
if self.theta == MAX_THETA {
return num_retained;
}
let theta = self.theta as f64 / MAX_THETA as f64;
num_retained / theta
}
pub fn theta(&self) -> f64 {
self.theta as f64 / MAX_THETA as f64
}
pub fn theta64(&self) -> u64 {
self.theta
}
pub fn is_empty(&self) -> bool {
self.empty
}
pub fn is_estimation_mode(&self) -> bool {
self.theta < MAX_THETA
}
pub fn num_retained(&self) -> usize {
self.entries.len()
}
pub fn is_ordered(&self) -> bool {
self.ordered
}
pub fn seed_hash(&self) -> u16 {
self.seed_hash
}
pub fn iter(&self) -> impl Iterator<Item = (u64, &S)> + '_ {
self.entries
.iter()
.map(|entry| (entry.hash(), entry.summary()))
}
pub fn lower_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::lower_bound(self.num_retained() as u64, self.theta(), num_std_dev)
.expect("compact theta should always be valid")
}
pub fn upper_bound(&self, num_std_dev: NumStdDev) -> f64 {
if !self.is_estimation_mode() {
return self.num_retained() as f64;
}
binomial_bounds::upper_bound(
self.num_retained() as u64,
self.theta(),
num_std_dev,
self.is_empty(),
)
.expect("compact theta should always be valid")
}
pub fn estimated_size(&self) -> usize {
size_of::<Self>() + self.entries.capacity() * size_of::<TupleEntry<S>>()
}
fn preamble_longs(&self) -> u8 {
if self.is_estimation_mode() {
3
} else if self.is_empty() || self.entries.len() == 1 {
1
} else {
2
}
}
pub fn serialize(&self) -> Vec<u8>
where
S: TupleSummaryValue,
{
let pre_longs = self.preamble_longs();
let entries_size: usize = self
.entries
.iter()
.map(|entry| 8 + entry.summary().serialize_size())
.sum();
let mut bytes = SketchBytes::with_capacity(8 * pre_longs as usize + entries_size);
bytes.write_u8(pre_longs);
bytes.write_u8(SERIAL_VERSION);
bytes.write_u8(Family::TUPLE.id);
bytes.write_u8(SKETCH_TYPE);
bytes.write_u8(0);
let mut flags = FLAGS_IS_READ_ONLY | FLAGS_IS_COMPACT;
if self.is_empty() {
flags |= FLAGS_IS_EMPTY;
}
if self.is_ordered() {
flags |= FLAGS_IS_ORDERED;
}
bytes.write_u8(flags);
bytes.write_u16_le(self.seed_hash);
if pre_longs > 1 {
bytes.write_u32_le(self.entries.len() as u32);
bytes.write_u32_le(0); }
if self.is_estimation_mode() {
bytes.write_u64_le(self.theta);
}
for entry in &self.entries {
bytes.write_u64_le(entry.hash());
entry.summary().serialize_value(&mut bytes);
}
bytes.into_bytes()
}
pub fn deserialize(bytes: &[u8]) -> Result<Self, Error>
where
S: TupleSummaryValue,
{
Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED)
}
pub fn deserialize_with_seed(bytes: &[u8], seed: u64) -> Result<Self, Error>
where
S: TupleSummaryValue,
{
let mut cursor = SketchSlice::new(bytes);
let pre_longs = cursor
.read_u8()
.map_err(insufficient_data("preamble_longs"))?;
let ser_ver = cursor
.read_u8()
.map_err(insufficient_data("serial_version"))?;
let family_id = cursor.read_u8().map_err(insufficient_data("family_id"))?;
let sketch_type = cursor.read_u8().map_err(insufficient_data("sketch_type"))?;
cursor.read_u8().map_err(insufficient_data("<unused>"))?;
let flags = cursor.read_u8().map_err(insufficient_data("flags"))?;
let seed_hash = cursor
.read_u16_le()
.map_err(insufficient_data("seed_hash"))?;
Family::TUPLE.validate_id(family_id)?;
ensure_preamble_longs_in_range(
Family::TUPLE.min_pre_longs..=Family::TUPLE.max_pre_longs,
pre_longs,
)?;
if ser_ver != SERIAL_VERSION && ser_ver != SERIAL_VERSION_LEGACY {
return Err(Error::deserial(format!(
"unsupported serial version: expected {} or {}, got {ser_ver}",
SERIAL_VERSION, SERIAL_VERSION_LEGACY,
)));
}
if sketch_type != SKETCH_TYPE && sketch_type != SKETCH_TYPE_LEGACY {
return Err(Error::deserial(format!(
"unsupported sketch type: expected {} or {}, got {sketch_type}",
SKETCH_TYPE, SKETCH_TYPE_LEGACY,
)));
}
let empty = (flags & FLAGS_IS_EMPTY) != 0;
let ordered = (flags & FLAGS_IS_ORDERED) != 0;
if empty {
return Ok(Self::from_parts(
vec![],
MAX_THETA,
seed_hash,
ordered,
true,
));
}
check_seed_hash(
compute_seed_hash(seed),
seed_hash,
"deserialized CompactTupleSketch",
ErrorKind::InvalidData,
)?;
let mut theta = MAX_THETA;
let num_entries = if pre_longs == 1 {
1usize
} else {
let n = cursor
.read_u32_le()
.map_err(insufficient_data("num_entries"))? as usize;
cursor
.read_u32_le()
.map_err(insufficient_data("<unused_u32>"))?;
if pre_longs > 2 {
theta = cursor.read_u64_le().map_err(insufficient_data("theta"))?;
}
n
};
let mut entries = Vec::with_capacity(num_entries);
for _ in 0..num_entries {
let hash = cursor
.read_u64_le()
.map_err(insufficient_data("entry_hash"))?;
if hash == 0 || hash >= theta {
return Err(Error::deserial("corrupted: invalid retained hash value"));
}
let summary = S::deserialize_value(&mut cursor)?;
entries.push(TupleEntry::new(hash, summary));
}
Ok(Self::from_parts(entries, theta, seed_hash, ordered, false))
}
}
#[derive(Debug)]
pub struct TupleSketchBuilder<P>
where
P: SummaryPolicy,
{
lg_k: u8,
resize_factor: ResizeFactor,
sampling_probability: f32,
seed: u64,
policy: P,
}
impl<P> TupleSketchBuilder<P>
where
P: SummaryPolicy,
{
pub fn new(policy: P) -> Self {
Self {
lg_k: DEFAULT_LG_K,
resize_factor: ResizeFactor::X8,
sampling_probability: 1.0,
seed: DEFAULT_UPDATE_SEED,
policy,
}
}
pub fn lg_k(mut self, lg_k: u8) -> Self {
assert!(
(MIN_LG_K..=MAX_LG_K).contains(&lg_k),
"lg_k must be in [{MIN_LG_K}, {MAX_LG_K}], got {lg_k}"
);
self.lg_k = lg_k;
self
}
pub fn resize_factor(mut self, factor: ResizeFactor) -> Self {
self.resize_factor = factor;
self
}
pub fn sampling_probability(mut self, probability: f32) -> Self {
assert!(
(0.0..=1.0).contains(&probability) && probability > 0.0,
"sampling_probability must be in (0.0, 1.0], got {probability}"
);
self.sampling_probability = probability;
self
}
pub fn seed(mut self, seed: u64) -> Self {
self.seed = seed;
self
}
pub fn build(self) -> TupleSketch<P> {
TupleSketch {
table: TupleHashTable::new(
self.lg_k,
self.resize_factor,
self.sampling_probability,
self.seed,
),
policy: self.policy,
}
}
}