recode/
recode.rs

1//! Re-encode a `.kv` through the writer and verify it round-trips, rebuilding the `.bt`
2//! (and `.kvei`, if a salt is given). Demonstrates the full write pipeline on real data.
3//!
4//! Usage: `cargo run --release --example recode -- <in.kv> <out.kv> [salt-state.txt]`
5
6use std::time::Instant;
7
8use erigon_seg::{DomainOptions, DomainWriter, KvReader, Salt, Seg, salt_from_file};
9
10fn main() -> Result<(), Box<dyn std::error::Error>> {
11    let mut args = std::env::args().skip(1);
12    let in_kv = args
13        .next()
14        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
15    let out_kv = args
16        .next()
17        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
18    let salt = args.next().and_then(salt_from_file);
19
20    // Read the source words (key/value pairs) and stream them into a DomainWriter.
21    let src = Seg::open(&in_kv)?;
22    let n_words = src.words_count();
23    println!("source: {} words ({} keys)", n_words, n_words / 2);
24
25    let t = Instant::now();
26    let mut w = DomainWriter::create(
27        &out_kv,
28        DomainOptions {
29            salt,
30            ..Default::default()
31        },
32    )?;
33    let mut g = src.getter();
34    while g.has_next() {
35        let key = g.next();
36        let value = if g.has_next() { g.next() } else { Vec::new() };
37        w.add(&key, &value)?;
38    }
39    let paths = w.finish()?;
40    println!("wrote {:?} in {:?}", paths, t.elapsed());
41
42    // Verify: every word matches the source byte-for-byte.
43    let dst = Seg::open(&out_kv)?;
44    assert_eq!(dst.words_count(), n_words);
45    let (mut a, mut b) = (src.getter(), dst.getter());
46    while a.has_next() {
47        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
48    }
49    assert!(!b.has_next());
50    println!("round-trip OK: all {n_words} words identical");
51
52    // If we built a bloom, confirm it accelerates lookups without false negatives.
53    if let Some(s) = salt {
54        let mut r = KvReader::open(&out_kv)?;
55        assert!(
56            r.enable_bloom(Salt::Known(s)),
57            "rebuilt .kvei failed to validate"
58        );
59        let mut checked = 0;
60        for kv in r.iter().step_by(997).take(500) {
61            let (k, v) = kv?;
62            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
63            checked += 1;
64        }
65        println!("bloom enabled; {checked} sampled lookups OK");
66    }
67
68    println!("\nOK");
69    Ok(())
70}
recode/recode.rs

recode/
recode.rs
