crush-core 0.3.0

High-performance parallel compression library core
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
//! Plugin metadata and file format structures

use crate::error::{Result, ValidationError};
use std::io::{Read, Write};

/// Metadata describing a compression plugin's capabilities and performance
#[derive(Debug, Clone, Copy)]
pub struct PluginMetadata {
    /// Plugin name (e.g., "deflate", "zstd", "lz4")
    pub name: &'static str,

    /// Plugin version (semantic versioning)
    pub version: &'static str,

    /// Unique 4-byte magic number for this plugin's compressed format
    /// Format: [0x43, 0x52, version, `plugin_id`]
    /// Where: 0x43='C', 0x52='R' (Crush identifier)
    pub magic_number: [u8; 4],

    /// Expected throughput in MB/s (measured under standard conditions)
    pub throughput: f64,

    /// Expected compression ratio (`compressed_size` / `original_size`)
    /// Range: (0.0, 1.0] where lower is better compression
    pub compression_ratio: f64,

    /// Human-readable description
    pub description: &'static str,
}

/// Crush compressed file header (16 bytes, little-endian)
///
/// Format:
/// ```text
/// Offset | Size | Field
/// -------|------|-------
/// 0      | 4    | magic_number ([u8; 4])
/// 4      | 8    | original_size (u64, little-endian)
/// 12     | 1    | flags (u8)
/// 13     | 3    | reserved (padding to 16 bytes)
/// ```
///
/// Flags byte (bit fields):
/// - Bit 0: Has CRC32 (if set, CRC32 follows header)
/// - Bit 1: Has metadata (if set, variable-length metadata follows header)
/// - Bits 2-7: Reserved for future use
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(C)]
pub struct CrushHeader {
    /// Magic number identifying the compression plugin
    pub magic: [u8; 4],

    /// Original uncompressed size in bytes
    pub original_size: u64,

    /// Feature flags (see struct documentation)
    pub flags: u8,

    /// Reserved bytes for future extensions (must be zero)
    pub reserved: [u8; 3],
}

/// Header feature flags
pub mod flags {
    /// CRC32 checksum present after header
    pub const HAS_CRC32: u8 = 0x01;

    /// Variable-length metadata section present
    pub const HAS_METADATA: u8 = 0x02;
}

impl CrushHeader {
    /// Size of the header in bytes (fixed at 16 bytes)
    pub const SIZE: usize = 16;

    /// Crush file format identifier prefix ("CR" = 0x43 0x52)
    pub const MAGIC_PREFIX: [u8; 2] = [0x43, 0x52];

    /// Crush format version (V1 = 0x01)
    pub const VERSION: u8 = 0x01;

    /// Create a new header with the given plugin magic number and original size
    #[must_use]
    pub fn new(magic: [u8; 4], original_size: u64) -> Self {
        Self {
            magic,
            flags: 0,
            original_size,
            reserved: [0; 3],
        }
    }

    /// Create a header with CRC32 flag set
    #[must_use]
    pub fn with_crc32(mut self) -> Self {
        self.flags |= flags::HAS_CRC32;
        self
    }

    /// Create a header with metadata flag set
    #[must_use]
    pub fn with_metadata(mut self) -> Self {
        self.flags |= flags::HAS_METADATA;
        self
    }

    /// Check if this header has a valid Crush magic number prefix
    #[must_use]
    pub fn has_valid_prefix(&self) -> bool {
        self.magic[0] == Self::MAGIC_PREFIX[0] && self.magic[1] == Self::MAGIC_PREFIX[1]
    }

    /// Check if this header has a valid Crush format version
    #[must_use]
    pub fn has_valid_version(&self) -> bool {
        self.magic[2] == Self::VERSION
    }

    /// Get the plugin ID from the magic number (4th byte)
    #[must_use]
    pub fn plugin_id(&self) -> u8 {
        self.magic[3]
    }

    /// Check if CRC32 flag is set
    #[must_use]
    pub fn has_crc32(&self) -> bool {
        (self.flags & flags::HAS_CRC32) != 0
    }

    /// Check if metadata flag is set
    #[must_use]
    pub fn has_metadata(&self) -> bool {
        (self.flags & flags::HAS_METADATA) != 0
    }

    /// Serialize header to bytes (little-endian)
    #[must_use]
    pub fn to_bytes(&self) -> [u8; Self::SIZE] {
        let mut bytes = [0u8; Self::SIZE];

        // Magic number (4 bytes)
        bytes[0..4].copy_from_slice(&self.magic);

        // Original size (8 bytes, little-endian)
        bytes[4..12].copy_from_slice(&self.original_size.to_le_bytes());

        // Flags (1 byte)
        bytes[12] = self.flags;

        // Reserved (3 bytes, must be zero)
        bytes[13..16].copy_from_slice(&self.reserved);

        bytes
    }

    /// Deserialize header from bytes
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - The magic number prefix is not valid (not "CR")
    /// - The version byte is unsupported
    pub fn from_bytes(bytes: &[u8; Self::SIZE]) -> Result<Self> {
        let header = Self {
            magic: [bytes[0], bytes[1], bytes[2], bytes[3]],
            original_size: u64::from_le_bytes([
                bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], bytes[11],
            ]),
            flags: bytes[12],
            reserved: [bytes[13], bytes[14], bytes[15]],
        };

        // Validate Crush format
        if !header.has_valid_prefix() {
            return Err(ValidationError::InvalidMagic(header.magic).into());
        }

        if !header.has_valid_version() {
            return Err(ValidationError::InvalidHeader(format!(
                "Unsupported version: 0x{:02x}",
                header.magic[2]
            ))
            .into());
        }

        Ok(header)
    }

    /// Write header to a writer
    ///
    /// # Errors
    ///
    /// Returns an error if the write operation fails
    pub fn write_to<W: Write>(&self, writer: &mut W) -> std::io::Result<()> {
        writer.write_all(&self.to_bytes())
    }

    /// Read header from a reader
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - The read operation fails
    /// - The header validation fails (invalid magic or version)
    pub fn read_from<R: Read>(reader: &mut R) -> Result<Self> {
        let mut bytes = [0u8; Self::SIZE];
        reader.read_exact(&mut bytes)?;
        Self::from_bytes(&bytes)
    }
}

use serde::Serialize; // This one should stay

/// Optional file metadata that can be stored in the compressed file
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize)]
pub struct FileMetadata {
    /// Modification time (seconds since Unix epoch)
    pub mtime: Option<i64>,

    /// Unix file permissions (mode bits)
    /// Only stored and restored on Unix platforms
    #[cfg(unix)]
    pub permissions: Option<u32>,
}

impl FileMetadata {
    /// Serialize metadata to a byte vector using TLV format
    #[must_use]
    pub fn to_bytes(&self) -> Vec<u8> {
        let mut bytes = Vec::new();
        if let Some(mtime) = self.mtime {
            // Type: 0x01 for mtime
            bytes.push(0x01);
            // Length: 8 bytes for i64
            bytes.push(8);
            // Value: mtime as i64
            bytes.extend_from_slice(&mtime.to_le_bytes());
        }
        #[cfg(unix)]
        if let Some(permissions) = self.permissions {
            // Type: 0x02 for Unix permissions
            bytes.push(0x02);
            // Length: 4 bytes for u32
            bytes.push(4);
            // Value: permissions as u32
            bytes.extend_from_slice(&permissions.to_le_bytes());
        }
        bytes
    }

    /// Deserialize metadata from bytes
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - The TLV record is incomplete or malformed
    /// - Unknown metadata type is encountered
    /// - Value length is incorrect for the type
    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
        let mut metadata = Self::default();
        let mut i = 0;
        while i < bytes.len() {
            if i + 2 > bytes.len() {
                return Err(
                    ValidationError::InvalidMetadata("Incomplete TLV record".into()).into(),
                );
            }
            let type_ = bytes[i];
            let length = bytes[i + 1] as usize;
            i += 2;

            if i + length > bytes.len() {
                return Err(ValidationError::InvalidMetadata("Incomplete TLV value".into()).into());
            }

            let value = &bytes[i..i + length];
            i += length;

            match type_ {
                0x01 => {
                    // mtime
                    if length == 8 {
                        let mut mtime_bytes = [0u8; 8];
                        mtime_bytes.copy_from_slice(value);
                        metadata.mtime = Some(i64::from_le_bytes(mtime_bytes));
                    } else {
                        return Err(ValidationError::InvalidMetadata(
                            "Invalid mtime length".into(),
                        )
                        .into());
                    }
                }
                #[cfg(unix)]
                0x02 => {
                    // Unix permissions
                    if length == 4 {
                        let mut perm_bytes = [0u8; 4];
                        perm_bytes.copy_from_slice(value);
                        metadata.permissions = Some(u32::from_le_bytes(perm_bytes));
                    } else {
                        return Err(ValidationError::InvalidMetadata(
                            "Invalid permissions length".into(),
                        )
                        .into());
                    }
                }
                _ => { /* Ignore unknown types for forward compatibility */ }
            }
        }
        Ok(metadata)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_header_size() {
        // The serialized header is always 16 bytes
        assert_eq!(CrushHeader::SIZE, 16);

        // Note: The in-memory struct size may be larger due to alignment padding
        // (typically 24 bytes with repr(C)), but serialization produces exactly 16 bytes
        let header = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 12345);
        assert_eq!(header.to_bytes().len(), 16);
    }

    #[test]
    #[allow(clippy::unwrap_used)]
    fn test_header_roundtrip() {
        let original = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 12345);
        let bytes = original.to_bytes();
        let deserialized = CrushHeader::from_bytes(&bytes).unwrap();

        assert_eq!(original, deserialized);
        assert_eq!(deserialized.original_size, 12345);
    }

    #[test]
    fn test_invalid_magic_prefix() {
        let mut bytes = [0u8; CrushHeader::SIZE];
        bytes[0] = 0xFF; // Invalid prefix
        bytes[1] = 0xFF;

        let result = CrushHeader::from_bytes(&bytes);
        assert!(result.is_err());
    }

    #[test]
    fn test_crc32_flag() {
        let header = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100).with_crc32();
        assert!(header.has_crc32());

        let bytes = header.to_bytes();
        assert_eq!(bytes[12] & flags::HAS_CRC32, flags::HAS_CRC32);
    }

    #[test]
    fn test_plugin_id() {
        let header1 = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100);
        assert_eq!(header1.plugin_id(), 0x00);

        let header2 = CrushHeader::new([0x43, 0x52, 0x01, 0xFF], 200);
        assert_eq!(header2.plugin_id(), 0xFF);

        let header3 = CrushHeader::new([0x43, 0x52, 0x01, 0x42], 300);
        assert_eq!(header3.plugin_id(), 0x42);
    }

    #[test]
    fn test_has_valid_prefix() {
        let valid = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100);
        assert!(valid.has_valid_prefix());

        // Invalid prefix
        let invalid = CrushHeader {
            magic: [0xFF, 0xFF, 0x01, 0x00],
            original_size: 100,
            flags: 0,
            reserved: [0; 3],
        };
        assert!(!invalid.has_valid_prefix());
    }

    #[test]
    fn test_has_valid_version() {
        let valid = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100);
        assert!(valid.has_valid_version());

        // Invalid version
        let invalid = CrushHeader {
            magic: [0x43, 0x52, 0x99, 0x00],
            original_size: 100,
            flags: 0,
            reserved: [0; 3],
        };
        assert!(!invalid.has_valid_version());
    }

    #[test]
    fn test_has_metadata_flag() {
        let without = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100);
        assert!(!without.has_metadata());

        let with = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100).with_metadata();
        assert!(with.has_metadata());

        let bytes = with.to_bytes();
        assert_eq!(bytes[12] & flags::HAS_METADATA, flags::HAS_METADATA);
    }

    #[test]
    fn test_combined_flags() {
        let header = CrushHeader::new([0x43, 0x52, 0x01, 0x00], 100)
            .with_crc32()
            .with_metadata();

        assert!(header.has_crc32());
        assert!(header.has_metadata());

        let bytes = header.to_bytes();
        assert_eq!(
            bytes[12] & (flags::HAS_CRC32 | flags::HAS_METADATA),
            flags::HAS_CRC32 | flags::HAS_METADATA
        );
    }
}