Struct KvReader 

pub struct KvReader { /* private fields */ }

A reader over one seg file set (.kv data, optional .bt index, optional .kvei existence filter).

Implementations

impl KvReader

pub fn open(kv_path: impl AsRef<Path>) -> Result<KvReader>

Open a .kv file and any sibling .bt / .kvei files found next to it (same base name). The existence filter is loaded but not used for lookups until a salt is supplied via enable_bloom.

Examples found in repository

examples/recode.rs (line 54)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let in_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let out_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let salt = args.next().and_then(salt_from_file);

    // Read the source words (key/value pairs) and stream them into a DomainWriter.
    let src = Seg::open(&in_kv)?;
    let n_words = src.words_count();
    println!("source: {} words ({} keys)", n_words, n_words / 2);

    let t = Instant::now();
    let mut w = DomainWriter::create(
        &out_kv,
        DomainOptions {
            salt,
            ..Default::default()
        },
    )?;
    let mut g = src.getter();
    while g.has_next() {
        let key = g.next();
        let value = if g.has_next() { g.next() } else { Vec::new() };
        w.add(&key, &value)?;
    }
    let paths = w.finish()?;
    println!("wrote {:?} in {:?}", paths, t.elapsed());

    // Verify: every word matches the source byte-for-byte.
    let dst = Seg::open(&out_kv)?;
    assert_eq!(dst.words_count(), n_words);
    let (mut a, mut b) = (src.getter(), dst.getter());
    while a.has_next() {
        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
    }
    assert!(!b.has_next());
    println!("round-trip OK: all {n_words} words identical");

    // If we built a bloom, confirm it accelerates lookups without false negatives.
    if let Some(s) = salt {
        let mut r = KvReader::open(&out_kv)?;
        assert!(
            r.enable_bloom(Salt::Known(s)),
            "rebuilt .kvei failed to validate"
        );
        let mut checked = 0;
        for kv in r.iter().step_by(997).take(500) {
            let (k, v) = kv?;
            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
            checked += 1;
        }
        println!("bloom enabled; {checked} sampled lookups OK");
    }

    println!("\nOK");
    Ok(())
}

examples/inspect.rs (line 23)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn open_with( kv_path: impl AsRef<Path>, opts: OpenOptions, ) -> Result<KvReader>

Like open but with explicit seg OpenOptions (e.g. for files carrying out-of-band metadata).

pub fn name(&self) -> &str

The .kv file’s base name (e.g. v1.1-accounts.0-1024.kv).

pub fn bloom_active(&self) -> bool

Whether the bloom filter is active for lookups — i.e. a .kvei is present and a salt has been validated against real keys via enable_bloom.

pub fn seg(&self) -> &Seg

The underlying seg data file.

Examples found in repository

examples/inspect.rs (line 25)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn index(&self) -> Option<&BtreeIndex>

The B-tree index, if a .bt was found.

Examples found in repository

examples/inspect.rs (line 29)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn existence_filter(&self) -> Option<&ExistenceFilter>

The existence filter, if a .kvei was found.

Examples found in repository

examples/inspect.rs (line 34)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn salt(&self) -> Option<u32>

The active bloom salt, if enable_bloom has succeeded.

Examples found in repository

examples/inspect.rs (line 134)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn key_count(&self) -> u64

Number of keys: from the .bt index if present, otherwise inferred as words_count / 2 (domain files store alternating key/value words).

Examples found in repository

examples/inspect.rs (line 28)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn enable_bloom(&mut self, salt: Salt) -> bool

Enable the .kvei bloom as a negative-lookup accelerator, resolving the salt per Salt. Returns true only if a usable bloom is present and the resolved salt self-validates against real keys (so a wrong salt can never cause a missed key — it just leaves lookups unaccelerated).

Examples found in repository

examples/recode.rs (line 56)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let in_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let out_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let salt = args.next().and_then(salt_from_file);

    // Read the source words (key/value pairs) and stream them into a DomainWriter.
    let src = Seg::open(&in_kv)?;
    let n_words = src.words_count();
    println!("source: {} words ({} keys)", n_words, n_words / 2);

    let t = Instant::now();
    let mut w = DomainWriter::create(
        &out_kv,
        DomainOptions {
            salt,
            ..Default::default()
        },
    )?;
    let mut g = src.getter();
    while g.has_next() {
        let key = g.next();
        let value = if g.has_next() { g.next() } else { Vec::new() };
        w.add(&key, &value)?;
    }
    let paths = w.finish()?;
    println!("wrote {:?} in {:?}", paths, t.elapsed());

    // Verify: every word matches the source byte-for-byte.
    let dst = Seg::open(&out_kv)?;
    assert_eq!(dst.words_count(), n_words);
    let (mut a, mut b) = (src.getter(), dst.getter());
    while a.has_next() {
        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
    }
    assert!(!b.has_next());
    println!("round-trip OK: all {n_words} words identical");

    // If we built a bloom, confirm it accelerates lookups without false negatives.
    if let Some(s) = salt {
        let mut r = KvReader::open(&out_kv)?;
        assert!(
            r.enable_bloom(Salt::Known(s)),
            "rebuilt .kvei failed to validate"
        );
        let mut checked = 0;
        for kv in r.iter().step_by(997).take(500) {
            let (k, v) = kv?;
            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
            checked += 1;
        }
        println!("bloom enabled; {checked} sampled lookups OK");
    }

    println!("\nOK");
    Ok(())
}

examples/inspect.rs (line 131)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn find_salt(&self, threads: usize) -> Option<u32>

Brute-force the bloom salt by requiring a batch of real keys to all hit the filter, using threads workers. Returns None if no .kvei bloom is usable or no salt validates (e.g. a fuse-filter or format mismatch).

Examples found in repository

examples/inspect.rs (line 112)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn advise_random(&self) -> Result<()>

Advise the kernel that this file set is read by point lookup, so a page fault should read one page instead of a read-ahead window.

A binary search touches a handful of scattered pages, and the kernel’s default fault-around then reads far more than is used — on a file much larger than RAM that read amplification dominates lookup latency. This is not the default, because suppressing read-ahead is a regression for a file small enough to sit in the page cache, where the surplus pages get used by later lookups anyway. Set it when the data is large relative to RAM; leave it alone otherwise.

Advice is a hint: errors are reported but ignoring them is safe, and on platforms without madvise this does nothing.

pub fn advise_sequential(&self) -> Result<()>

Advise the kernel that this .kv is about to be read front to back — before an iter or a merge — so read-ahead works in your favour.

pub fn index_bytes(&self) -> u64

Bytes preload_index would make resident: the .bt, plus the .kvei when the bloom is active. Use it to budget before calling.

pub fn preload_index(&self) -> u64

Read the index files into the page cache and return once they are resident, reporting how many bytes were loaded.

A point lookup touches the .bt far more than the .kv — every search comparison reads the Elias-Fano offset array, while only the final block of keys is decompressed — and the .bt is one to two orders of magnitude smaller. On a machine with RAM to spare, holding the whole index resident removes nearly all the remaining faults: on a 37 GiB file set with a 1.4 GiB .bt, cold lookups went from ~440 µs to ~155 µs, for a one-off ~0.7 s load.

The .kvei is included only when the bloom is active, since otherwise it is never read. Loading is a hint to the kernel, not a reservation: these pages can still be evicted under memory pressure — see lock_index to prevent that.

pub fn lock_index(&self) -> Result<()>

Pin the index files in RAM with mlock, so the kernel cannot evict them.

Stronger than preload_index, and worth it when a large .kv is streaming through the page cache and would otherwise push the index back out. Same file selection: the .bt, plus the .kvei when the bloom is active.

Fails with ENOMEM (or EPERM) if the total exceeds RLIMIT_MEMLOCK, which is commonly a few megabytes by default; check index_bytes against ulimit -l first. A failure is safe to ignore — it just leaves the pages evictable — but note the limit applies per process across every locked mapping.

Unix only: elsewhere this reports ErrorKind::Unsupported rather than quietly doing nothing, since the point of the call is a guarantee. preload_index still works everywhere.

Preloads first: mlock faults the pages in itself, but page-at-a-time, so warming them sequentially beforehand is markedly faster.

pub fn unlock_index(&self) -> Result<()>

Release the pages pinned by lock_index.

pub fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>>

Look up key, returning its value if present.

Uses, in order: the bloom filter for a fast definite-absent answer (if enabled), then the .bt index for an O(log n) binary search, or — if there is no index — an ordered linear scan.

Examples found in repository

examples/recode.rs (line 62)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let in_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let out_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let salt = args.next().and_then(salt_from_file);

    // Read the source words (key/value pairs) and stream them into a DomainWriter.
    let src = Seg::open(&in_kv)?;
    let n_words = src.words_count();
    println!("source: {} words ({} keys)", n_words, n_words / 2);

    let t = Instant::now();
    let mut w = DomainWriter::create(
        &out_kv,
        DomainOptions {
            salt,
            ..Default::default()
        },
    )?;
    let mut g = src.getter();
    while g.has_next() {
        let key = g.next();
        let value = if g.has_next() { g.next() } else { Vec::new() };
        w.add(&key, &value)?;
    }
    let paths = w.finish()?;
    println!("wrote {:?} in {:?}", paths, t.elapsed());

    // Verify: every word matches the source byte-for-byte.
    let dst = Seg::open(&out_kv)?;
    assert_eq!(dst.words_count(), n_words);
    let (mut a, mut b) = (src.getter(), dst.getter());
    while a.has_next() {
        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
    }
    assert!(!b.has_next());
    println!("round-trip OK: all {n_words} words identical");

    // If we built a bloom, confirm it accelerates lookups without false negatives.
    if let Some(s) = salt {
        let mut r = KvReader::open(&out_kv)?;
        assert!(
            r.enable_bloom(Salt::Known(s)),
            "rebuilt .kvei failed to validate"
        );
        let mut checked = 0;
        for kv in r.iter().step_by(997).take(500) {
            let (k, v) = kv?;
            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
            checked += 1;
        }
        println!("bloom enabled; {checked} sampled lookups OK");
    }

    println!("\nOK");
    Ok(())
}

examples/inspect.rs (line 77)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}

pub fn iter(&self) -> KvIter<'_>

Iterate every (key, value) pair sequentially, in stored (key) order.

Examples found in repository

examples/recode.rs (line 60)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let in_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let out_kv = args
        .next()
        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
    let salt = args.next().and_then(salt_from_file);

    // Read the source words (key/value pairs) and stream them into a DomainWriter.
    let src = Seg::open(&in_kv)?;
    let n_words = src.words_count();
    println!("source: {} words ({} keys)", n_words, n_words / 2);

    let t = Instant::now();
    let mut w = DomainWriter::create(
        &out_kv,
        DomainOptions {
            salt,
            ..Default::default()
        },
    )?;
    let mut g = src.getter();
    while g.has_next() {
        let key = g.next();
        let value = if g.has_next() { g.next() } else { Vec::new() };
        w.add(&key, &value)?;
    }
    let paths = w.finish()?;
    println!("wrote {:?} in {:?}", paths, t.elapsed());

    // Verify: every word matches the source byte-for-byte.
    let dst = Seg::open(&out_kv)?;
    assert_eq!(dst.words_count(), n_words);
    let (mut a, mut b) = (src.getter(), dst.getter());
    while a.has_next() {
        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
    }
    assert!(!b.has_next());
    println!("round-trip OK: all {n_words} words identical");

    // If we built a bloom, confirm it accelerates lookups without false negatives.
    if let Some(s) = salt {
        let mut r = KvReader::open(&out_kv)?;
        assert!(
            r.enable_bloom(Salt::Known(s)),
            "rebuilt .kvei failed to validate"
        );
        let mut checked = 0;
        for kv in r.iter().step_by(997).take(500) {
            let (k, v) = kv?;
            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
            checked += 1;
        }
        println!("bloom enabled; {checked} sampled lookups OK");
    }

    println!("\nOK");
    Ok(())
}

examples/inspect.rs (line 46)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = std::env::args().skip(1);
    let kv_path = args
        .next()
        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
    let salt_path = args.next();

    let t = Instant::now();
    let mut r = KvReader::open(&kv_path)?;
    println!("opened {kv_path} in {:?}", t.elapsed());
    println!("  seg version      : v{}", r.seg().version());
    println!("  words_count      : {}", r.seg().words_count());
    println!("  empty_words      : {}", r.seg().empty_words_count());
    println!("  key_count        : {}", r.key_count());
    println!("  has .bt index    : {}", r.index().is_some());
    if let Some(idx) = r.index() {
        println!("    bt key_count   : {}", idx.key_count());
        println!("    bt M           : {:?}", idx.m());
    }
    match r.existence_filter() {
        Some(f) => println!(
            "  .kvei kind       : {:?} (accelerating={})",
            f.kind(),
            f.is_accelerating()
        ),
        None => println!("  .kvei            : (none)"),
    }

    // First few (key, value) pairs.
    println!("\nfirst pairs:");
    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
    for (i, kv) in r.iter().enumerate().take(5) {
        let (k, v) = kv?;
        println!(
            "  [{i}] key={} ({} B)  value={} B",
            hex(&k),
            k.len(),
            v.len()
        );
        sample.push((k, v));
    }

    // Round-trip a spread of real keys through get().
    println!("\nround-trip lookups (spread across the file):");
    let n = r.key_count();
    let mut checked = 0u64;
    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
    if let Some(idx) = r.index() {
        let g_count = 12u64.min(n.max(1));
        let mut g = r.seg().getter();
        for s in 0..g_count {
            let di = s * n / g_count;
            if let Some(off) = idx.key_offset(di) {
                g.reset(off);
                probe_keys.push(g.next());
            }
        }
    } else {
        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
    }
    let t = Instant::now();
    for k in &probe_keys {
        let got = r.get(k)?;
        assert!(got.is_some(), "real key {} not found by get()", hex(k));
        checked += 1;
    }
    let dt = t.elapsed();
    println!(
        "  {checked} keys all found; avg {:?}/lookup",
        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
    );

    // Cross-check: get() value equals the value that follows the key in iteration order.
    for (k, v) in &sample {
        assert_eq!(
            r.get(k)?.as_deref(),
            Some(v.as_slice()),
            "value mismatch for {}",
            hex(k)
        );
    }
    println!("  values match sequential iteration ✓");

    // Negative lookup: a key we are confident is absent.
    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
    println!(
        "\nnegative lookup for a synthetic key: {:?}",
        r.get(absent)?.map(|v| v.len())
    );

    // Salt resolution + bloom acceleration.
    if r.existence_filter()
        .map(|f| f.is_accelerating())
        .unwrap_or(false)
    {
        // (a) brute-force find.
        let t = Instant::now();
        let found = r.find_salt(num_cpus());
        println!(
            "\nfind_salt -> {:?}  (in {:?})",
            found.map(|s| format!("{s:#010x}")),
            t.elapsed()
        );

        // (b) known salt from the salt file, if provided.
        let salt = match &salt_path {
            Some(p) => salt_from_file(p),
            None => None,
        };
        if let Some(s) = salt {
            println!("salt-state.txt   -> {s:#010x}");
            if let Some(f) = found {
                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
            }
        }
        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
        let enabled = r.enable_bloom(chosen);
        println!(
            "enable_bloom     -> {enabled} (active salt = {:?})",
            r.salt().map(|s| format!("{s:#010x}"))
        );

        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
            // Every real probe key must be reported present by the bloom.
            let all_present = probe_keys
                .iter()
                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
            println!(
                "bloom: all {} real probe keys reported present = {all_present}",
                probe_keys.len()
            );
            assert!(
                all_present,
                "bloom false-negative on a real key (wrong salt?)"
            );

            // Timed lookups with bloom enabled (negatives short-circuit).
            let t = Instant::now();
            for k in &probe_keys {
                let _ = r.get(k)?;
            }
            println!(
                "  {} bloom-gated lookups in {:?}",
                probe_keys.len(),
                t.elapsed()
            );
        }
    }

    println!("\nOK");
    Ok(())
}