hll/
hll.rs

1//! Standalone demo of the HLL (HyperLogLog) sketch and union APIs.
2//!
3//! Run with: `cargo run -p apache-datasketches --example hll`
4
5use apache_datasketches::hll::{HllSketch, HllUnion, TargetHllType};
6
7fn main() {
8    // A sketch estimates the number of distinct items seen, using bounded
9    // memory regardless of how many items are added. `lg_config_k` (4..=21)
10    // trades memory for accuracy: higher values are more accurate but use
11    // more space.
12    let mut sketch = HllSketch::new(12, TargetHllType::Hll4).expect("valid lg_config_k");
13
14    for i in 0..10_000u64 {
15        sketch.update_u64(i);
16    }
17    // Duplicates don't affect the distinct count.
18    for i in 0..5_000u64 {
19        sketch.update_u64(i);
20    }
21    sketch.update_str("some-key");
22    sketch.update_bytes(b"raw bytes work too");
23
24    println!("distinct count estimate: {:.1}", sketch.get_estimate());
25    println!(
26        "95% confidence interval: [{:.1}, {:.1}]",
27        sketch.get_lower_bound(2).unwrap(),
28        sketch.get_upper_bound(2).unwrap()
29    );
30
31    // Sketches can be serialized to bytes (e.g. to store or send over the
32    // network) and reconstructed later.
33    let bytes = sketch.serialize_compact();
34    let restored = HllSketch::deserialize(&bytes).expect("valid sketch bytes");
35    assert_eq!(sketch.get_estimate(), restored.get_estimate());
36    println!("serialized to {} bytes and restored successfully", bytes.len());
37
38    // HllUnion merges multiple sketches into one, e.g. combining
39    // per-shard/per-day counts into a total distinct count.
40    let mut shard_a = HllSketch::new(12, TargetHllType::Hll4).unwrap();
41    for i in 0..10_000u64 {
42        shard_a.update_u64(i);
43    }
44    let mut shard_b = HllSketch::new(12, TargetHllType::Hll4).unwrap();
45    for i in 5_000..15_000u64 {
46        shard_b.update_u64(i);
47    }
48
49    let mut union = HllUnion::new(12).expect("valid lg_max_k");
50    union.update_sketch(&shard_a);
51    union.update_sketch(&shard_b);
52
53    let merged = union.get_result(TargetHllType::Hll4);
54    println!(
55        "merged distinct count across two overlapping shards (true count = 15000): {:.1}",
56        merged.get_estimate()
57    );
58}
hll/hll.rs

hll/
hll.rs
