contextdb_server/
chunking.rs

1use crate::protocol::ChunkMessage;
2
3pub const CHUNK_SIZE: usize = 512 * 1024;
4pub const CHUNKING_THRESHOLD: usize = 900 * 1024;
5
6pub fn needs_chunking(data: &[u8]) -> bool {
7    data.len() > CHUNKING_THRESHOLD
8}
9
10pub fn chunk(data: &[u8]) -> Vec<ChunkMessage> {
11    let chunk_id = uuid::Uuid::new_v4();
12    let total = data.len().div_ceil(CHUNK_SIZE);
13    data.chunks(CHUNK_SIZE)
14        .enumerate()
15        .map(|(i, slice)| ChunkMessage {
16            chunk_id,
17            sequence: i as u32,
18            total_chunks: total as u32,
19            payload: slice.to_vec(),
20        })
21        .collect()
22}
23
24pub fn reassemble(chunks: &mut [ChunkMessage]) -> Vec<u8> {
25    chunks.sort_by_key(|c| c.sequence);
26    let mut bytes = Vec::new();
27    for chunk in chunks {
28        bytes.extend_from_slice(&chunk.payload);
29    }
30    bytes
31}
contextdb_server/chunking.rs

contextdb_server/
chunking.rs
