lencode 0.1.6

A crate for encoding and decoding variable-length data using the Lencode varint encoding scheme and deduplication
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

#[cfg(feature = "solana")]
use lencode::{
    dedupe::{DedupeDecoder, DedupeEncoder},
    prelude::*,
};
#[cfg(feature = "solana")]
use rand::Rng;
#[cfg(feature = "solana")]
use std::io::Cursor;

#[cfg(feature = "solana")]
use solana_pubkey::Pubkey;

#[cfg(feature = "solana")]
fn main() {
    // Create a vector of 1000 pubkeys where 50% are duplicates
    let mut rng = rand::rng();
    let unique_pubkeys: Vec<Pubkey> = (0..500).map(|_| Pubkey::new_unique()).collect();

    // Create duplicates by randomly selecting from unique pubkeys
    let duplicates: Vec<Pubkey> = (0..500)
        .map(|_| {
            let idx = rng.random_range(0..unique_pubkeys.len());
            unique_pubkeys[idx]
        })
        .collect();

    // Combine vectors
    let mut all_pubkeys = unique_pubkeys;
    all_pubkeys.extend(duplicates);

    // Encode with borsh
    let borsh_data = borsh::to_vec(&all_pubkeys).unwrap();

    // Encode with lencode + deduplication
    let mut encoder = DedupeEncoder::with_capacity(1000, 1);
    let mut cursor = Cursor::new(Vec::new());
    all_pubkeys
        .encode_ext(&mut cursor, Some(&mut encoder))
        .unwrap();
    let lencode_data = cursor.into_inner();

    println!("Vector size: {} pubkeys", all_pubkeys.len());
    println!("Borsh encoded size: {} bytes", borsh_data.len());
    println!("Lencode encoded size: {} bytes", lencode_data.len());
    println!(
        "Space savings: {:.1}%",
        100.0 * (1.0 - lencode_data.len() as f64 / borsh_data.len() as f64)
    );
    println!(
        "Unique values stored: {} out of {} total",
        encoder.len(),
        all_pubkeys.len()
    );

    // Verify we can decode correctly
    let mut decoder = DedupeDecoder::with_capacity(1000);
    let mut cursor = Cursor::new(&lencode_data);
    let decoded: Vec<Pubkey> = Vec::decode_ext(&mut cursor, Some(&mut decoder)).unwrap();

    assert_eq!(all_pubkeys.len(), decoded.len());
    assert_eq!(all_pubkeys, decoded);
    println!("✓ Decoding verification passed");
}

#[cfg(not(feature = "solana"))]
fn main() {
    println!("This example requires the 'solana' feature to be enabled.");
    println!("Run with: cargo run --example size_comparison --features=solana");
}