Struct HllUnion 

pub struct HllUnion { /* private fields */ }

Merges multiple HllSketches into one, e.g. combining per-shard or per-day counts into a total distinct count across all of them.

Implementations

impl HllUnion

pub fn new(lg_max_k: u8) -> Result<Self, SketchError>

Creates a new, empty union with the given maximum lg_config_k (4..=21) — the union’s result will use at most this lg_config_k, even if a merged-in sketch used a larger one. Returns SketchError::InvalidConfig if lg_max_k is out of range.

Examples found in repository

examples/hll.rs (line 52)

fn main() {
    // A sketch estimates the number of distinct items seen, using bounded
    // memory regardless of how many items are added. `lg_config_k` (4..=21)
    // trades memory for accuracy: higher values are more accurate but use
    // more space.
    let mut sketch = HllSketch::new(12, TargetHllType::Hll4).expect("valid lg_config_k");

    for i in 0..10_000u64 {
        sketch.update_u64(i);
    }
    // Duplicates don't affect the distinct count.
    for i in 0..5_000u64 {
        sketch.update_u64(i);
    }
    sketch.update_str("some-key");
    sketch.update_bytes(b"raw bytes work too");

    println!("distinct count estimate: {:.1}", sketch.get_estimate());
    println!(
        "95% confidence interval: [{:.1}, {:.1}]",
        sketch.get_lower_bound(2).unwrap(),
        sketch.get_upper_bound(2).unwrap()
    );

    // Sketches can be serialized to bytes (e.g. to store or send over the
    // network) and reconstructed later.
    let bytes = sketch.serialize_compact();
    let restored = HllSketch::deserialize(&bytes).expect("valid sketch bytes");
    assert_eq!(sketch.get_estimate(), restored.get_estimate());
    println!(
        "serialized to {} bytes and restored successfully",
        bytes.len()
    );

    // HllUnion merges multiple sketches into one, e.g. combining
    // per-shard/per-day counts into a total distinct count.
    let mut shard_a = HllSketch::new(12, TargetHllType::Hll4).unwrap();
    for i in 0..10_000u64 {
        shard_a.update_u64(i);
    }
    let mut shard_b = HllSketch::new(12, TargetHllType::Hll4).unwrap();
    for i in 5_000..15_000u64 {
        shard_b.update_u64(i);
    }

    let mut union = HllUnion::new(12).expect("valid lg_max_k");
    union.update_sketch(&shard_a);
    union.update_sketch(&shard_b);

    let merged = union.get_result(TargetHllType::Hll4);
    println!(
        "merged distinct count across two overlapping shards (true count = 15000): {:.1}",
        merged.get_estimate()
    );
}

pub fn update_sketch(&mut self, sketch: &HllSketch)

Merges the given sketch’s state into this union.

Examples found in repository

examples/hll.rs (line 53)

fn main() {
    // A sketch estimates the number of distinct items seen, using bounded
    // memory regardless of how many items are added. `lg_config_k` (4..=21)
    // trades memory for accuracy: higher values are more accurate but use
    // more space.
    let mut sketch = HllSketch::new(12, TargetHllType::Hll4).expect("valid lg_config_k");

    for i in 0..10_000u64 {
        sketch.update_u64(i);
    }
    // Duplicates don't affect the distinct count.
    for i in 0..5_000u64 {
        sketch.update_u64(i);
    }
    sketch.update_str("some-key");
    sketch.update_bytes(b"raw bytes work too");

    println!("distinct count estimate: {:.1}", sketch.get_estimate());
    println!(
        "95% confidence interval: [{:.1}, {:.1}]",
        sketch.get_lower_bound(2).unwrap(),
        sketch.get_upper_bound(2).unwrap()
    );

    // Sketches can be serialized to bytes (e.g. to store or send over the
    // network) and reconstructed later.
    let bytes = sketch.serialize_compact();
    let restored = HllSketch::deserialize(&bytes).expect("valid sketch bytes");
    assert_eq!(sketch.get_estimate(), restored.get_estimate());
    println!(
        "serialized to {} bytes and restored successfully",
        bytes.len()
    );

    // HllUnion merges multiple sketches into one, e.g. combining
    // per-shard/per-day counts into a total distinct count.
    let mut shard_a = HllSketch::new(12, TargetHllType::Hll4).unwrap();
    for i in 0..10_000u64 {
        shard_a.update_u64(i);
    }
    let mut shard_b = HllSketch::new(12, TargetHllType::Hll4).unwrap();
    for i in 5_000..15_000u64 {
        shard_b.update_u64(i);
    }

    let mut union = HllUnion::new(12).expect("valid lg_max_k");
    union.update_sketch(&shard_a);
    union.update_sketch(&shard_b);

    let merged = union.get_result(TargetHllType::Hll4);
    println!(
        "merged distinct count across two overlapping shards (true count = 15000): {:.1}",
        merged.get_estimate()
    );
}

pub fn update_u64(&mut self, value: u64)

Adds a u64 value directly to the union, as if it were added to every sketch merged into it.

pub fn update_i64(&mut self, value: i64)

Adds an i64 value directly to the union. See Self::update_u64.

pub fn update_f64(&mut self, value: f64)

Adds an f64 value directly to the union. See Self::update_u64.

pub fn update_str(&mut self, value: &str)

Adds a string value directly to the union. See Self::update_u64.

pub fn update_bytes(&mut self, value: &[u8])

Adds an arbitrary byte slice directly to the union. See Self::update_u64.

pub fn get_result(&self, tgt_type: TargetHllType) -> HllSketch

Returns the current merged result as a standalone HllSketch of the given TargetHllType.

Examples found in repository

examples/hll.rs (line 56)

fn main() {
    // A sketch estimates the number of distinct items seen, using bounded
    // memory regardless of how many items are added. `lg_config_k` (4..=21)
    // trades memory for accuracy: higher values are more accurate but use
    // more space.
    let mut sketch = HllSketch::new(12, TargetHllType::Hll4).expect("valid lg_config_k");

    for i in 0..10_000u64 {
        sketch.update_u64(i);
    }
    // Duplicates don't affect the distinct count.
    for i in 0..5_000u64 {
        sketch.update_u64(i);
    }
    sketch.update_str("some-key");
    sketch.update_bytes(b"raw bytes work too");

    println!("distinct count estimate: {:.1}", sketch.get_estimate());
    println!(
        "95% confidence interval: [{:.1}, {:.1}]",
        sketch.get_lower_bound(2).unwrap(),
        sketch.get_upper_bound(2).unwrap()
    );

    // Sketches can be serialized to bytes (e.g. to store or send over the
    // network) and reconstructed later.
    let bytes = sketch.serialize_compact();
    let restored = HllSketch::deserialize(&bytes).expect("valid sketch bytes");
    assert_eq!(sketch.get_estimate(), restored.get_estimate());
    println!(
        "serialized to {} bytes and restored successfully",
        bytes.len()
    );

    // HllUnion merges multiple sketches into one, e.g. combining
    // per-shard/per-day counts into a total distinct count.
    let mut shard_a = HllSketch::new(12, TargetHllType::Hll4).unwrap();
    for i in 0..10_000u64 {
        shard_a.update_u64(i);
    }
    let mut shard_b = HllSketch::new(12, TargetHllType::Hll4).unwrap();
    for i in 5_000..15_000u64 {
        shard_b.update_u64(i);
    }

    let mut union = HllUnion::new(12).expect("valid lg_max_k");
    union.update_sketch(&shard_a);
    union.update_sketch(&shard_b);

    let merged = union.get_result(TargetHllType::Hll4);
    println!(
        "merged distinct count across two overlapping shards (true count = 15000): {:.1}",
        merged.get_estimate()
    );
}

pub fn serialize_compact(&self, tgt_type: TargetHllType) -> Vec<u8>

Serializes get_result(tgt_type) in compact form. A union has no serializable state of its own upstream (only the result sketch does) — to resume accumulating after deserializing, use HllSketch::deserialize and feed the sketch back in via update_sketch.

pub fn serialize_updatable(&self, tgt_type: TargetHllType) -> Vec<u8>

Serializes get_result(tgt_type) in updatable form. See Self::serialize_compact for why HllUnion has no deserialize.

pub fn get_estimate(&self) -> f64

Returns the current estimate of the number of distinct items merged into this union so far.

pub fn get_lower_bound(&self, num_std_dev: u8) -> Result<f64, SketchError>

Returns the lower bound of the confidence interval around Self::get_estimate. See HllSketch::get_lower_bound for the meaning of num_std_dev.

pub fn get_upper_bound(&self, num_std_dev: u8) -> Result<f64, SketchError>

Returns the upper bound of the confidence interval around Self::get_estimate. See HllSketch::get_lower_bound for the meaning of num_std_dev.

pub fn is_empty(&self) -> bool

Returns true if no sketch or item has been merged into this union.

pub fn reset(&mut self)

Resets this union to its initial, empty state.

Trait Implementations

impl Send for HllUnion