hll/
hll.rs

1//! Standalone demo of the HLL (HyperLogLog) sketch and union APIs.
2//!
3//! Run with: `cargo run -p apache-datasketches --example hll`
4
5use apache_datasketches::hll::{HllSketch, HllUnion, TargetHllType};
6
7fn main() {
8    // A sketch estimates the number of distinct items seen, using bounded
9    // memory regardless of how many items are added. `lg_config_k` (4..=21)
10    // trades memory for accuracy: higher values are more accurate but use
11    // more space.
12    let mut sketch = HllSketch::new(12, TargetHllType::Hll4).expect("valid lg_config_k");
13
14    for i in 0..10_000u64 {
15        sketch.update_u64(i);
16    }
17    // Duplicates don't affect the distinct count.
18    for i in 0..5_000u64 {
19        sketch.update_u64(i);
20    }
21    sketch.update_str("some-key");
22    sketch.update_bytes(b"raw bytes work too");
23
24    println!("distinct count estimate: {:.1}", sketch.get_estimate());
25    println!(
26        "95% confidence interval: [{:.1}, {:.1}]",
27        sketch.get_lower_bound(2).unwrap(),
28        sketch.get_upper_bound(2).unwrap()
29    );
30
31    // Sketches can be serialized to bytes (e.g. to store or send over the
32    // network) and reconstructed later.
33    let bytes = sketch.serialize_compact();
34    let restored = HllSketch::deserialize(&bytes).expect("valid sketch bytes");
35    assert_eq!(sketch.get_estimate(), restored.get_estimate());
36    println!(
37        "serialized to {} bytes and restored successfully",
38        bytes.len()
39    );
40
41    // HllUnion merges multiple sketches into one, e.g. combining
42    // per-shard/per-day counts into a total distinct count.
43    let mut shard_a = HllSketch::new(12, TargetHllType::Hll4).unwrap();
44    for i in 0..10_000u64 {
45        shard_a.update_u64(i);
46    }
47    let mut shard_b = HllSketch::new(12, TargetHllType::Hll4).unwrap();
48    for i in 5_000..15_000u64 {
49        shard_b.update_u64(i);
50    }
51
52    let mut union = HllUnion::new(12).expect("valid lg_max_k");
53    union.update_sketch(&shard_a);
54    union.update_sketch(&shard_b);
55
56    let merged = union.get_result(TargetHllType::Hll4);
57    println!(
58        "merged distinct count across two overlapping shards (true count = 15000): {:.1}",
59        merged.get_estimate()
60    );
61}
hll/hll.rs

hll/
hll.rs
