lib-compression 0.1.0

Universal lossless compression via network-wide deduplication
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

//! Sovereign Compression Library
//!
//! Universal lossless compression via network-wide deduplication with optional zero-knowledge proofs.
//!
//! # Features
//!
//! - **Content-defined chunking** (FastCDC)
//! - **Pattern-based deduplication** (ZKC)
//! - **SovereignCodec** (BWT + MTF + RLE + Range coding)
//! - **Optional ZK proofs** (feature: `zk-proofs`)
//! - **Optional ML compression** (feature: `ml-support`)
//!
//! # Example
//!
//! ```rust
//! use lib_compression::{ContentChunker, ZkcCompressor, SovereignCodec};
//!
//! // Chunk file
//! let chunker = ContentChunker::default();
//! let shards = chunker.chunk(&data)?;
//!
//! // Compress with ZKC
//! let compressor = ZkcCompressor::new();
//! let compressed = compressor.compress(&shards)?;
//! ```

// Core modules
pub mod chunker;
pub mod shard;
pub mod assembler;
pub mod transport;
pub mod error;

// ZKC modules
pub mod patterns;
pub mod pattern_dict;
pub mod zkc_compressor;
pub mod zkc_decompressor;
pub mod sovereign_codec;

// Abstraction traits (decouples from internal workspace deps)
pub mod traits;

// Optional model compressor bridge
pub mod model_compressor;

// Re-exports
pub use chunker::ContentChunker;
pub use shard::{Shard, ShardId, ShardManager};
pub use assembler::JitAssembler;
pub use error::{CompressionError, Result};
pub use transport::{ShardTransport, TransportConfig};
pub use patterns::{Pattern, PatternId, PatternMiner, PatternMinerConfig};
pub use pattern_dict::{PatternDictionary, DictionaryEntry};
pub use zkc_compressor::{ZkcCompressor, CompressedShard, CompressionStats};
pub use zkc_decompressor::{ZkcDecompressor, DecompressionStats};
pub use sovereign_codec::{SovereignCodec, CodecParams};
pub use model_compressor::{SovereignCodecCompressor, AdaptiveCodecCompressor};

// Re-export traits for easy access
pub use traits::{ProofBackend, DhtStorage, ModelCompressor};
pub use traits::{DefaultProofBackend, MemoryDht, DefaultModelCompressor};
pub use traits::MerkleProof;

// Constants
pub const PROTOCOL_VERSION: u8 = 1;
pub const AVG_SHARD_SIZE: usize = 1024 * 1024; // 1 MB
pub const MIN_SHARD_SIZE: usize = 256 * 1024;  // 256 KB
pub const MAX_SHARD_SIZE: usize = 4 * 1024 * 1024; // 4 MB
pub const DEFAULT_REDUNDANCY: usize = 3;