Skip to main content

Seg

Struct Seg 

Source
pub struct Seg { /* private fields */ }
Expand description

A seg-compressed file (.kv). Owns its mmap and dictionaries; create a Getter to read words.

Implementations§

Source§

impl Seg

Source

pub fn open(path: impl AsRef<Path>) -> Result<Seg>

Open a .kv with default options (no out-of-band metadata).

Examples found in repository?
examples/recode.rs (line 21)
10fn main() -> Result<(), Box<dyn std::error::Error>> {
11    let mut args = std::env::args().skip(1);
12    let in_kv = args
13        .next()
14        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
15    let out_kv = args
16        .next()
17        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
18    let salt = args.next().and_then(salt_from_file);
19
20    // Read the source words (key/value pairs) and stream them into a DomainWriter.
21    let src = Seg::open(&in_kv)?;
22    let n_words = src.words_count();
23    println!("source: {} words ({} keys)", n_words, n_words / 2);
24
25    let t = Instant::now();
26    let mut w = DomainWriter::create(
27        &out_kv,
28        DomainOptions {
29            salt,
30            ..Default::default()
31        },
32    )?;
33    let mut g = src.getter();
34    while g.has_next() {
35        let key = g.next();
36        let value = if g.has_next() { g.next() } else { Vec::new() };
37        w.add(&key, &value)?;
38    }
39    let paths = w.finish()?;
40    println!("wrote {:?} in {:?}", paths, t.elapsed());
41
42    // Verify: every word matches the source byte-for-byte.
43    let dst = Seg::open(&out_kv)?;
44    assert_eq!(dst.words_count(), n_words);
45    let (mut a, mut b) = (src.getter(), dst.getter());
46    while a.has_next() {
47        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
48    }
49    assert!(!b.has_next());
50    println!("round-trip OK: all {n_words} words identical");
51
52    // If we built a bloom, confirm it accelerates lookups without false negatives.
53    if let Some(s) = salt {
54        let mut r = KvReader::open(&out_kv)?;
55        assert!(
56            r.enable_bloom(Salt::Known(s)),
57            "rebuilt .kvei failed to validate"
58        );
59        let mut checked = 0;
60        for kv in r.iter().step_by(997).take(500) {
61            let (k, v) = kv?;
62            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
63            checked += 1;
64        }
65        println!("bloom enabled; {checked} sampled lookups OK");
66    }
67
68    println!("\nOK");
69    Ok(())
70}
Source

pub fn open_with(path: impl AsRef<Path>, opts: OpenOptions) -> Result<Seg>

Open a .kv with explicit OpenOptions.

Source

pub fn words_count(&self) -> u64

Number of words in the file (for a domain .kv, 2 × key_count).

Examples found in repository?
examples/recode.rs (line 22)
10fn main() -> Result<(), Box<dyn std::error::Error>> {
11    let mut args = std::env::args().skip(1);
12    let in_kv = args
13        .next()
14        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
15    let out_kv = args
16        .next()
17        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
18    let salt = args.next().and_then(salt_from_file);
19
20    // Read the source words (key/value pairs) and stream them into a DomainWriter.
21    let src = Seg::open(&in_kv)?;
22    let n_words = src.words_count();
23    println!("source: {} words ({} keys)", n_words, n_words / 2);
24
25    let t = Instant::now();
26    let mut w = DomainWriter::create(
27        &out_kv,
28        DomainOptions {
29            salt,
30            ..Default::default()
31        },
32    )?;
33    let mut g = src.getter();
34    while g.has_next() {
35        let key = g.next();
36        let value = if g.has_next() { g.next() } else { Vec::new() };
37        w.add(&key, &value)?;
38    }
39    let paths = w.finish()?;
40    println!("wrote {:?} in {:?}", paths, t.elapsed());
41
42    // Verify: every word matches the source byte-for-byte.
43    let dst = Seg::open(&out_kv)?;
44    assert_eq!(dst.words_count(), n_words);
45    let (mut a, mut b) = (src.getter(), dst.getter());
46    while a.has_next() {
47        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
48    }
49    assert!(!b.has_next());
50    println!("round-trip OK: all {n_words} words identical");
51
52    // If we built a bloom, confirm it accelerates lookups without false negatives.
53    if let Some(s) = salt {
54        let mut r = KvReader::open(&out_kv)?;
55        assert!(
56            r.enable_bloom(Salt::Known(s)),
57            "rebuilt .kvei failed to validate"
58        );
59        let mut checked = 0;
60        for kv in r.iter().step_by(997).take(500) {
61            let (k, v) = kv?;
62            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
63            checked += 1;
64        }
65        println!("bloom enabled; {checked} sampled lookups OK");
66    }
67
68    println!("\nOK");
69    Ok(())
70}
More examples
Hide additional examples
examples/inspect.rs (line 26)
15fn main() -> Result<(), Box<dyn std::error::Error>> {
16    let mut args = std::env::args().skip(1);
17    let kv_path = args
18        .next()
19        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
20    let salt_path = args.next();
21
22    let t = Instant::now();
23    let mut r = KvReader::open(&kv_path)?;
24    println!("opened {kv_path} in {:?}", t.elapsed());
25    println!("  seg version      : v{}", r.seg().version());
26    println!("  words_count      : {}", r.seg().words_count());
27    println!("  empty_words      : {}", r.seg().empty_words_count());
28    println!("  key_count        : {}", r.key_count());
29    println!("  has .bt index    : {}", r.index().is_some());
30    if let Some(idx) = r.index() {
31        println!("    bt key_count   : {}", idx.key_count());
32        println!("    bt M           : {:?}", idx.m());
33    }
34    match r.existence_filter() {
35        Some(f) => println!(
36            "  .kvei kind       : {:?} (accelerating={})",
37            f.kind(),
38            f.is_accelerating()
39        ),
40        None => println!("  .kvei            : (none)"),
41    }
42
43    // First few (key, value) pairs.
44    println!("\nfirst pairs:");
45    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
46    for (i, kv) in r.iter().enumerate().take(5) {
47        let (k, v) = kv?;
48        println!(
49            "  [{i}] key={} ({} B)  value={} B",
50            hex(&k),
51            k.len(),
52            v.len()
53        );
54        sample.push((k, v));
55    }
56
57    // Round-trip a spread of real keys through get().
58    println!("\nround-trip lookups (spread across the file):");
59    let n = r.key_count();
60    let mut checked = 0u64;
61    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
62    if let Some(idx) = r.index() {
63        let g_count = 12u64.min(n.max(1));
64        let mut g = r.seg().getter();
65        for s in 0..g_count {
66            let di = s * n / g_count;
67            if let Some(off) = idx.key_offset(di) {
68                g.reset(off);
69                probe_keys.push(g.next());
70            }
71        }
72    } else {
73        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
74    }
75    let t = Instant::now();
76    for k in &probe_keys {
77        let got = r.get(k)?;
78        assert!(got.is_some(), "real key {} not found by get()", hex(k));
79        checked += 1;
80    }
81    let dt = t.elapsed();
82    println!(
83        "  {checked} keys all found; avg {:?}/lookup",
84        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
85    );
86
87    // Cross-check: get() value equals the value that follows the key in iteration order.
88    for (k, v) in &sample {
89        assert_eq!(
90            r.get(k)?.as_deref(),
91            Some(v.as_slice()),
92            "value mismatch for {}",
93            hex(k)
94        );
95    }
96    println!("  values match sequential iteration ✓");
97
98    // Negative lookup: a key we are confident is absent.
99    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
100    println!(
101        "\nnegative lookup for a synthetic key: {:?}",
102        r.get(absent)?.map(|v| v.len())
103    );
104
105    // Salt resolution + bloom acceleration.
106    if r.existence_filter()
107        .map(|f| f.is_accelerating())
108        .unwrap_or(false)
109    {
110        // (a) brute-force find.
111        let t = Instant::now();
112        let found = r.find_salt(num_cpus());
113        println!(
114            "\nfind_salt -> {:?}  (in {:?})",
115            found.map(|s| format!("{s:#010x}")),
116            t.elapsed()
117        );
118
119        // (b) known salt from the salt file, if provided.
120        let salt = match &salt_path {
121            Some(p) => salt_from_file(p),
122            None => None,
123        };
124        if let Some(s) = salt {
125            println!("salt-state.txt   -> {s:#010x}");
126            if let Some(f) = found {
127                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
128            }
129        }
130        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
131        let enabled = r.enable_bloom(chosen);
132        println!(
133            "enable_bloom     -> {enabled} (active salt = {:?})",
134            r.salt().map(|s| format!("{s:#010x}"))
135        );
136
137        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
138            // Every real probe key must be reported present by the bloom.
139            let all_present = probe_keys
140                .iter()
141                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
142            println!(
143                "bloom: all {} real probe keys reported present = {all_present}",
144                probe_keys.len()
145            );
146            assert!(
147                all_present,
148                "bloom false-negative on a real key (wrong salt?)"
149            );
150
151            // Timed lookups with bloom enabled (negatives short-circuit).
152            let t = Instant::now();
153            for k in &probe_keys {
154                let _ = r.get(k)?;
155            }
156            println!(
157                "  {} bloom-gated lookups in {:?}",
158                probe_keys.len(),
159                t.elapsed()
160            );
161        }
162    }
163
164    println!("\nOK");
165    Ok(())
166}
Source

pub fn empty_words_count(&self) -> u64

Number of empty (zero-length) words.

Examples found in repository?
examples/inspect.rs (line 27)
15fn main() -> Result<(), Box<dyn std::error::Error>> {
16    let mut args = std::env::args().skip(1);
17    let kv_path = args
18        .next()
19        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
20    let salt_path = args.next();
21
22    let t = Instant::now();
23    let mut r = KvReader::open(&kv_path)?;
24    println!("opened {kv_path} in {:?}", t.elapsed());
25    println!("  seg version      : v{}", r.seg().version());
26    println!("  words_count      : {}", r.seg().words_count());
27    println!("  empty_words      : {}", r.seg().empty_words_count());
28    println!("  key_count        : {}", r.key_count());
29    println!("  has .bt index    : {}", r.index().is_some());
30    if let Some(idx) = r.index() {
31        println!("    bt key_count   : {}", idx.key_count());
32        println!("    bt M           : {:?}", idx.m());
33    }
34    match r.existence_filter() {
35        Some(f) => println!(
36            "  .kvei kind       : {:?} (accelerating={})",
37            f.kind(),
38            f.is_accelerating()
39        ),
40        None => println!("  .kvei            : (none)"),
41    }
42
43    // First few (key, value) pairs.
44    println!("\nfirst pairs:");
45    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
46    for (i, kv) in r.iter().enumerate().take(5) {
47        let (k, v) = kv?;
48        println!(
49            "  [{i}] key={} ({} B)  value={} B",
50            hex(&k),
51            k.len(),
52            v.len()
53        );
54        sample.push((k, v));
55    }
56
57    // Round-trip a spread of real keys through get().
58    println!("\nround-trip lookups (spread across the file):");
59    let n = r.key_count();
60    let mut checked = 0u64;
61    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
62    if let Some(idx) = r.index() {
63        let g_count = 12u64.min(n.max(1));
64        let mut g = r.seg().getter();
65        for s in 0..g_count {
66            let di = s * n / g_count;
67            if let Some(off) = idx.key_offset(di) {
68                g.reset(off);
69                probe_keys.push(g.next());
70            }
71        }
72    } else {
73        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
74    }
75    let t = Instant::now();
76    for k in &probe_keys {
77        let got = r.get(k)?;
78        assert!(got.is_some(), "real key {} not found by get()", hex(k));
79        checked += 1;
80    }
81    let dt = t.elapsed();
82    println!(
83        "  {checked} keys all found; avg {:?}/lookup",
84        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
85    );
86
87    // Cross-check: get() value equals the value that follows the key in iteration order.
88    for (k, v) in &sample {
89        assert_eq!(
90            r.get(k)?.as_deref(),
91            Some(v.as_slice()),
92            "value mismatch for {}",
93            hex(k)
94        );
95    }
96    println!("  values match sequential iteration ✓");
97
98    // Negative lookup: a key we are confident is absent.
99    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
100    println!(
101        "\nnegative lookup for a synthetic key: {:?}",
102        r.get(absent)?.map(|v| v.len())
103    );
104
105    // Salt resolution + bloom acceleration.
106    if r.existence_filter()
107        .map(|f| f.is_accelerating())
108        .unwrap_or(false)
109    {
110        // (a) brute-force find.
111        let t = Instant::now();
112        let found = r.find_salt(num_cpus());
113        println!(
114            "\nfind_salt -> {:?}  (in {:?})",
115            found.map(|s| format!("{s:#010x}")),
116            t.elapsed()
117        );
118
119        // (b) known salt from the salt file, if provided.
120        let salt = match &salt_path {
121            Some(p) => salt_from_file(p),
122            None => None,
123        };
124        if let Some(s) = salt {
125            println!("salt-state.txt   -> {s:#010x}");
126            if let Some(f) = found {
127                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
128            }
129        }
130        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
131        let enabled = r.enable_bloom(chosen);
132        println!(
133            "enable_bloom     -> {enabled} (active salt = {:?})",
134            r.salt().map(|s| format!("{s:#010x}"))
135        );
136
137        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
138            // Every real probe key must be reported present by the bloom.
139            let all_present = probe_keys
140                .iter()
141                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
142            println!(
143                "bloom: all {} real probe keys reported present = {all_present}",
144                probe_keys.len()
145            );
146            assert!(
147                all_present,
148                "bloom false-negative on a real key (wrong salt?)"
149            );
150
151            // Timed lookups with bloom enabled (negatives short-circuit).
152            let t = Instant::now();
153            for k in &probe_keys {
154                let _ = r.get(k)?;
155            }
156            println!(
157                "  {} bloom-gated lookups in {:?}",
158                probe_keys.len(),
159                t.elapsed()
160            );
161        }
162    }
163
164    println!("\nOK");
165    Ok(())
166}
Source

pub fn version(&self) -> u8

On-disk header version (0 = legacy, 1 = versioned header).

Examples found in repository?
examples/inspect.rs (line 25)
15fn main() -> Result<(), Box<dyn std::error::Error>> {
16    let mut args = std::env::args().skip(1);
17    let kv_path = args
18        .next()
19        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
20    let salt_path = args.next();
21
22    let t = Instant::now();
23    let mut r = KvReader::open(&kv_path)?;
24    println!("opened {kv_path} in {:?}", t.elapsed());
25    println!("  seg version      : v{}", r.seg().version());
26    println!("  words_count      : {}", r.seg().words_count());
27    println!("  empty_words      : {}", r.seg().empty_words_count());
28    println!("  key_count        : {}", r.key_count());
29    println!("  has .bt index    : {}", r.index().is_some());
30    if let Some(idx) = r.index() {
31        println!("    bt key_count   : {}", idx.key_count());
32        println!("    bt M           : {:?}", idx.m());
33    }
34    match r.existence_filter() {
35        Some(f) => println!(
36            "  .kvei kind       : {:?} (accelerating={})",
37            f.kind(),
38            f.is_accelerating()
39        ),
40        None => println!("  .kvei            : (none)"),
41    }
42
43    // First few (key, value) pairs.
44    println!("\nfirst pairs:");
45    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
46    for (i, kv) in r.iter().enumerate().take(5) {
47        let (k, v) = kv?;
48        println!(
49            "  [{i}] key={} ({} B)  value={} B",
50            hex(&k),
51            k.len(),
52            v.len()
53        );
54        sample.push((k, v));
55    }
56
57    // Round-trip a spread of real keys through get().
58    println!("\nround-trip lookups (spread across the file):");
59    let n = r.key_count();
60    let mut checked = 0u64;
61    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
62    if let Some(idx) = r.index() {
63        let g_count = 12u64.min(n.max(1));
64        let mut g = r.seg().getter();
65        for s in 0..g_count {
66            let di = s * n / g_count;
67            if let Some(off) = idx.key_offset(di) {
68                g.reset(off);
69                probe_keys.push(g.next());
70            }
71        }
72    } else {
73        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
74    }
75    let t = Instant::now();
76    for k in &probe_keys {
77        let got = r.get(k)?;
78        assert!(got.is_some(), "real key {} not found by get()", hex(k));
79        checked += 1;
80    }
81    let dt = t.elapsed();
82    println!(
83        "  {checked} keys all found; avg {:?}/lookup",
84        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
85    );
86
87    // Cross-check: get() value equals the value that follows the key in iteration order.
88    for (k, v) in &sample {
89        assert_eq!(
90            r.get(k)?.as_deref(),
91            Some(v.as_slice()),
92            "value mismatch for {}",
93            hex(k)
94        );
95    }
96    println!("  values match sequential iteration ✓");
97
98    // Negative lookup: a key we are confident is absent.
99    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
100    println!(
101        "\nnegative lookup for a synthetic key: {:?}",
102        r.get(absent)?.map(|v| v.len())
103    );
104
105    // Salt resolution + bloom acceleration.
106    if r.existence_filter()
107        .map(|f| f.is_accelerating())
108        .unwrap_or(false)
109    {
110        // (a) brute-force find.
111        let t = Instant::now();
112        let found = r.find_salt(num_cpus());
113        println!(
114            "\nfind_salt -> {:?}  (in {:?})",
115            found.map(|s| format!("{s:#010x}")),
116            t.elapsed()
117        );
118
119        // (b) known salt from the salt file, if provided.
120        let salt = match &salt_path {
121            Some(p) => salt_from_file(p),
122            None => None,
123        };
124        if let Some(s) = salt {
125            println!("salt-state.txt   -> {s:#010x}");
126            if let Some(f) = found {
127                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
128            }
129        }
130        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
131        let enabled = r.enable_bloom(chosen);
132        println!(
133            "enable_bloom     -> {enabled} (active salt = {:?})",
134            r.salt().map(|s| format!("{s:#010x}"))
135        );
136
137        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
138            // Every real probe key must be reported present by the bloom.
139            let all_present = probe_keys
140                .iter()
141                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
142            println!(
143                "bloom: all {} real probe keys reported present = {all_present}",
144                probe_keys.len()
145            );
146            assert!(
147                all_present,
148                "bloom false-negative on a real key (wrong salt?)"
149            );
150
151            // Timed lookups with bloom enabled (negatives short-circuit).
152            let t = Instant::now();
153            for k in &probe_keys {
154                let _ = r.get(k)?;
155            }
156            println!(
157                "  {} bloom-gated lookups in {:?}",
158                probe_keys.len(),
159                t.elapsed()
160            );
161        }
162    }
163
164    println!("\nOK");
165    Ok(())
166}
Source

pub fn page_values_count(&self) -> u8

Values-per-page if page-level compression is enabled, else 0. Page reassembly is not performed by this low-level reader; words are returned as encoded.

Source

pub fn advise_random(&self) -> Result<()>

Advise the kernel that this .kv is read in random order (point lookups). See KvReader::advise_random.

Source

pub fn advise_sequential(&self) -> Result<()>

Advise the kernel that this .kv is read front to back, so read-ahead helps. Worth setting before an iter or a merge.

Source

pub fn len(&self) -> usize

Total mapped file length in bytes — a safe over-estimate of the maximum word offset, suitable as the max_offset bound when building a .bt index.

Source

pub fn is_empty(&self) -> bool

Whether the file maps to zero bytes.

Source

pub fn getter(&self) -> Getter<'_>

Create a cursor positioned at the start of the words region.

Examples found in repository?
examples/recode.rs (line 33)
10fn main() -> Result<(), Box<dyn std::error::Error>> {
11    let mut args = std::env::args().skip(1);
12    let in_kv = args
13        .next()
14        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
15    let out_kv = args
16        .next()
17        .expect("usage: recode <in.kv> <out.kv> [salt-state.txt]");
18    let salt = args.next().and_then(salt_from_file);
19
20    // Read the source words (key/value pairs) and stream them into a DomainWriter.
21    let src = Seg::open(&in_kv)?;
22    let n_words = src.words_count();
23    println!("source: {} words ({} keys)", n_words, n_words / 2);
24
25    let t = Instant::now();
26    let mut w = DomainWriter::create(
27        &out_kv,
28        DomainOptions {
29            salt,
30            ..Default::default()
31        },
32    )?;
33    let mut g = src.getter();
34    while g.has_next() {
35        let key = g.next();
36        let value = if g.has_next() { g.next() } else { Vec::new() };
37        w.add(&key, &value)?;
38    }
39    let paths = w.finish()?;
40    println!("wrote {:?} in {:?}", paths, t.elapsed());
41
42    // Verify: every word matches the source byte-for-byte.
43    let dst = Seg::open(&out_kv)?;
44    assert_eq!(dst.words_count(), n_words);
45    let (mut a, mut b) = (src.getter(), dst.getter());
46    while a.has_next() {
47        assert_eq!(a.next(), b.next(), "word mismatch after re-encode");
48    }
49    assert!(!b.has_next());
50    println!("round-trip OK: all {n_words} words identical");
51
52    // If we built a bloom, confirm it accelerates lookups without false negatives.
53    if let Some(s) = salt {
54        let mut r = KvReader::open(&out_kv)?;
55        assert!(
56            r.enable_bloom(Salt::Known(s)),
57            "rebuilt .kvei failed to validate"
58        );
59        let mut checked = 0;
60        for kv in r.iter().step_by(997).take(500) {
61            let (k, v) = kv?;
62            assert_eq!(r.get(&k)?.as_deref(), Some(v.as_slice()));
63            checked += 1;
64        }
65        println!("bloom enabled; {checked} sampled lookups OK");
66    }
67
68    println!("\nOK");
69    Ok(())
70}
More examples
Hide additional examples
examples/inspect.rs (line 64)
15fn main() -> Result<(), Box<dyn std::error::Error>> {
16    let mut args = std::env::args().skip(1);
17    let kv_path = args
18        .next()
19        .expect("usage: inspect <path-to.kv> [salt-state.txt]");
20    let salt_path = args.next();
21
22    let t = Instant::now();
23    let mut r = KvReader::open(&kv_path)?;
24    println!("opened {kv_path} in {:?}", t.elapsed());
25    println!("  seg version      : v{}", r.seg().version());
26    println!("  words_count      : {}", r.seg().words_count());
27    println!("  empty_words      : {}", r.seg().empty_words_count());
28    println!("  key_count        : {}", r.key_count());
29    println!("  has .bt index    : {}", r.index().is_some());
30    if let Some(idx) = r.index() {
31        println!("    bt key_count   : {}", idx.key_count());
32        println!("    bt M           : {:?}", idx.m());
33    }
34    match r.existence_filter() {
35        Some(f) => println!(
36            "  .kvei kind       : {:?} (accelerating={})",
37            f.kind(),
38            f.is_accelerating()
39        ),
40        None => println!("  .kvei            : (none)"),
41    }
42
43    // First few (key, value) pairs.
44    println!("\nfirst pairs:");
45    let mut sample: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
46    for (i, kv) in r.iter().enumerate().take(5) {
47        let (k, v) = kv?;
48        println!(
49            "  [{i}] key={} ({} B)  value={} B",
50            hex(&k),
51            k.len(),
52            v.len()
53        );
54        sample.push((k, v));
55    }
56
57    // Round-trip a spread of real keys through get().
58    println!("\nround-trip lookups (spread across the file):");
59    let n = r.key_count();
60    let mut checked = 0u64;
61    let mut probe_keys: Vec<Vec<u8>> = Vec::new();
62    if let Some(idx) = r.index() {
63        let g_count = 12u64.min(n.max(1));
64        let mut g = r.seg().getter();
65        for s in 0..g_count {
66            let di = s * n / g_count;
67            if let Some(off) = idx.key_offset(di) {
68                g.reset(off);
69                probe_keys.push(g.next());
70            }
71        }
72    } else {
73        probe_keys = sample.iter().map(|(k, _)| k.clone()).collect();
74    }
75    let t = Instant::now();
76    for k in &probe_keys {
77        let got = r.get(k)?;
78        assert!(got.is_some(), "real key {} not found by get()", hex(k));
79        checked += 1;
80    }
81    let dt = t.elapsed();
82    println!(
83        "  {checked} keys all found; avg {:?}/lookup",
84        dt.checked_div(checked.max(1) as u32).unwrap_or_default()
85    );
86
87    // Cross-check: get() value equals the value that follows the key in iteration order.
88    for (k, v) in &sample {
89        assert_eq!(
90            r.get(k)?.as_deref(),
91            Some(v.as_slice()),
92            "value mismatch for {}",
93            hex(k)
94        );
95    }
96    println!("  values match sequential iteration ✓");
97
98    // Negative lookup: a key we are confident is absent.
99    let absent = b"\xff_erigon_seg_definitely_absent_key_\xff";
100    println!(
101        "\nnegative lookup for a synthetic key: {:?}",
102        r.get(absent)?.map(|v| v.len())
103    );
104
105    // Salt resolution + bloom acceleration.
106    if r.existence_filter()
107        .map(|f| f.is_accelerating())
108        .unwrap_or(false)
109    {
110        // (a) brute-force find.
111        let t = Instant::now();
112        let found = r.find_salt(num_cpus());
113        println!(
114            "\nfind_salt -> {:?}  (in {:?})",
115            found.map(|s| format!("{s:#010x}")),
116            t.elapsed()
117        );
118
119        // (b) known salt from the salt file, if provided.
120        let salt = match &salt_path {
121            Some(p) => salt_from_file(p),
122            None => None,
123        };
124        if let Some(s) = salt {
125            println!("salt-state.txt   -> {s:#010x}");
126            if let Some(f) = found {
127                assert_eq!(f, s, "brute-forced salt disagrees with salt file");
128            }
129        }
130        let chosen = salt.map(Salt::Known).unwrap_or(Salt::Find(num_cpus()));
131        let enabled = r.enable_bloom(chosen);
132        println!(
133            "enable_bloom     -> {enabled} (active salt = {:?})",
134            r.salt().map(|s| format!("{s:#010x}"))
135        );
136
137        if let (Some(s), Some(f)) = (r.salt(), r.existence_filter()) {
138            // Every real probe key must be reported present by the bloom.
139            let all_present = probe_keys
140                .iter()
141                .all(|k| f.contains_hash(murmur3_x64_128_h1(k, s)));
142            println!(
143                "bloom: all {} real probe keys reported present = {all_present}",
144                probe_keys.len()
145            );
146            assert!(
147                all_present,
148                "bloom false-negative on a real key (wrong salt?)"
149            );
150
151            // Timed lookups with bloom enabled (negatives short-circuit).
152            let t = Instant::now();
153            for k in &probe_keys {
154                let _ = r.get(k)?;
155            }
156            println!(
157                "  {} bloom-gated lookups in {:?}",
158                probe_keys.len(),
159                t.elapsed()
160            );
161        }
162    }
163
164    println!("\nOK");
165    Ok(())
166}

Auto Trait Implementations§

§

impl Freeze for Seg

§

impl RefUnwindSafe for Seg

§

impl Send for Seg

§

impl Sync for Seg

§

impl Unpin for Seg

§

impl UnsafeUnpin for Seg

§

impl UnwindSafe for Seg

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = Infallible

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.