prolly/prolly/
chunking.rs

1//! Built-in persisted chunking policies.
2
3use super::encoding::{
4    DEFAULT_CHUNKING_FACTOR, DEFAULT_HASH_SEED, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MIN_CHUNK_SIZE,
5};
6use super::format::{BoundaryInput, BoundaryRule, ChunkMeasure, ChunkingSpec, HashAlgorithm};
7
8/// Entry-count chunking with boundaries derived from keys and values.
9pub fn entry_count_key_value_hash() -> ChunkingSpec {
10    ChunkingSpec {
11        measure: ChunkMeasure::EntryCount,
12        input: BoundaryInput::KeyValue,
13        hash: HashAlgorithm::XxHash64,
14        rule: BoundaryRule::HashThreshold {
15            factor: DEFAULT_CHUNKING_FACTOR,
16        },
17        min: DEFAULT_MIN_CHUNK_SIZE as u64,
18        target: DEFAULT_CHUNKING_FACTOR as u64,
19        max: DEFAULT_MAX_CHUNK_SIZE as u64,
20        hash_seed: DEFAULT_HASH_SEED,
21        level_salt: false,
22        hard_max_node_bytes: 16 * 1024 * 1024,
23    }
24}
25
26/// Entry-count chunking with value-stable, key-only boundaries.
27pub fn entry_count_key_hash() -> ChunkingSpec {
28    ChunkingSpec::default()
29}
30
31/// Logical-byte chunking with key-only bounded Weibull boundaries.
32pub fn logical_bytes_key_weibull() -> ChunkingSpec {
33    ChunkingSpec {
34        measure: ChunkMeasure::LogicalBytes,
35        input: BoundaryInput::Key,
36        hash: HashAlgorithm::XxHash64,
37        rule: BoundaryRule::Weibull { shape: 2 },
38        min: 4 * 1024,
39        target: 16 * 1024,
40        max: 64 * 1024,
41        hash_seed: DEFAULT_HASH_SEED,
42        level_salt: true,
43        hard_max_node_bytes: 16 * 1024 * 1024,
44    }
45}
46
47/// Logical-byte chunking with a rolling content hash.
48pub fn logical_bytes_rolling_hash() -> ChunkingSpec {
49    ChunkingSpec {
50        measure: ChunkMeasure::LogicalBytes,
51        input: BoundaryInput::Key,
52        hash: HashAlgorithm::XxHash64,
53        rule: BoundaryRule::RollingBuzHash { window: 64 },
54        min: 4 * 1024,
55        target: 16 * 1024,
56        max: 64 * 1024,
57        hash_seed: DEFAULT_HASH_SEED,
58        level_salt: true,
59        hard_max_node_bytes: 16 * 1024 * 1024,
60    }
61}
prolly/prolly/chunking.rs

prolly/prolly/
chunking.rs
