shadow_crypt_core/
archive.rs

1//! The shadow archive payload format: a directory tree serialized as one
2//! byte stream.
3//!
4//! An archive travels as the *content* of an encrypted file whose metadata
5//! envelope marks it as [`crate::file::ContentKind::Archive`], so it
6//! inherits the container format's encryption, authentication, and
7//! streaming. This module defines only the plaintext payload layout and is
8//! deliberately independent of the format versions: the layout carries its
9//! own magic and version byte and can be reused by future container
10//! versions unchanged.
11//!
12//! Layout (all integers little endian):
13//!
14//! ```text
15//! magic: "SHDWARC" + version byte 1        (8 bytes)
16//! entries, each:
17//!   entry_type: u8                          (1 = file, 2 = directory, 0 = end)
18//!   for file/directory entries:
19//!     path_len: u16, path: UTF-8            ('/'-separated relative path)
20//!     flags: u8                             (bit 0 mtime, bit 1 mode)
21//!     mtime_secs: i64, mtime_nanos: u32     (if flag)
22//!     mode: u32                             (if flag)
23//!   for file entries:
24//!     content_len: u64, content bytes
25//! terminator: entry_type 0; nothing may follow
26//! ```
27//!
28//! Directory entries appear before their contents. Paths are validated on
29//! both encode and parse: relative, '/'-separated, no `..` or `.` or empty
30//! components, no backslashes.
31//!
32//! Encoding is a set of pure functions producing header bytes (the caller
33//! interleaves raw file content); parsing is the incremental
34//! [`ArchiveParser`], fed arbitrary byte pieces and drained of
35//! [`ArchiveEvent`]s, so neither side ever needs the whole archive in
36//! memory.
37
38use std::time::{Duration, SystemTime, UNIX_EPOCH};
39
40use zeroize::Zeroizing;
41
42use crate::{
43    file::FileMetadata,
44    memory::{SecureBytes, SecureString},
45};
46
47pub const MAGIC: [u8; 8] = *b"SHDWARC\x01";
48
49const ENTRY_END: u8 = 0;
50const ENTRY_FILE: u8 = 1;
51const ENTRY_DIR: u8 = 2;
52
53const FLAG_MTIME: u8 = 0b0000_0001;
54const FLAG_MODE: u8 = 0b0000_0010;
55
56/// Upper bound on an entry path, matching common filesystem limits.
57pub const MAX_PATH_LEN: usize = 4096;
58
59/// Errors from encoding or parsing an archive stream.
60#[derive(Debug)]
61pub enum ArchiveError {
62    /// A path is empty, absolute, contains `..`/`.`/empty components or
63    /// backslashes, or exceeds [`MAX_PATH_LEN`].
64    InvalidPath,
65    /// The stream is structurally malformed (bad magic, unknown entry type
66    /// or flags, invalid field values).
67    InvalidData,
68    /// The stream ended before the terminator entry.
69    Truncated,
70    /// Data follows the terminator entry.
71    TrailingData,
72}
73
74impl std::fmt::Display for ArchiveError {
75    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76        match self {
77            ArchiveError::InvalidPath => write!(f, "Archive entry path is invalid"),
78            ArchiveError::InvalidData => write!(f, "Archive stream is malformed"),
79            ArchiveError::Truncated => write!(f, "Archive stream ended unexpectedly"),
80            ArchiveError::TrailingData => write!(f, "Data present after the archive terminator"),
81        }
82    }
83}
84
85impl std::error::Error for ArchiveError {}
86
87/// Validates a '/'-separated relative entry path.
88pub fn validate_path(path: &str) -> Result<(), ArchiveError> {
89    if path.is_empty() || path.len() > MAX_PATH_LEN || path.contains('\\') {
90        return Err(ArchiveError::InvalidPath);
91    }
92    for component in path.split('/') {
93        if component.is_empty() || component == "." || component == ".." {
94            return Err(ArchiveError::InvalidPath);
95        }
96    }
97    Ok(())
98}
99
100fn encode_entry_header(entry_type: u8, metadata: &FileMetadata) -> Result<Vec<u8>, ArchiveError> {
101    let path = metadata.filename().as_str();
102    validate_path(path)?;
103
104    let mtime = metadata.mtime().map(systemtime_to_parts);
105
106    let mut flags = 0u8;
107    if mtime.is_some() {
108        flags |= FLAG_MTIME;
109    }
110    if metadata.mode().is_some() {
111        flags |= FLAG_MODE;
112    }
113
114    let mut bytes = Vec::with_capacity(1 + 2 + path.len() + 1 + 12 + 4);
115    bytes.push(entry_type);
116    bytes.extend_from_slice(&(path.len() as u16).to_le_bytes());
117    bytes.extend_from_slice(path.as_bytes());
118    bytes.push(flags);
119    if let Some((secs, nanos)) = mtime {
120        bytes.extend_from_slice(&secs.to_le_bytes());
121        bytes.extend_from_slice(&nanos.to_le_bytes());
122    }
123    if let Some(mode) = metadata.mode() {
124        bytes.extend_from_slice(&mode.to_le_bytes());
125    }
126    Ok(bytes)
127}
128
129/// Header bytes for a directory entry. `metadata.filename()` is the
130/// directory's relative path.
131pub fn encode_directory(metadata: &FileMetadata) -> Result<Vec<u8>, ArchiveError> {
132    encode_entry_header(ENTRY_DIR, metadata)
133}
134
135/// Header bytes for a file entry; exactly `content_len` raw content bytes
136/// must follow. `metadata.filename()` is the file's relative path.
137pub fn encode_file(metadata: &FileMetadata, content_len: u64) -> Result<Vec<u8>, ArchiveError> {
138    let mut bytes = encode_entry_header(ENTRY_FILE, metadata)?;
139    bytes.extend_from_slice(&content_len.to_le_bytes());
140    Ok(bytes)
141}
142
143/// The archive terminator entry.
144pub fn encode_end() -> [u8; 1] {
145    [ENTRY_END]
146}
147
148/// One parsed element of an archive stream, in stream order.
149#[derive(Debug)]
150pub enum ArchiveEvent {
151    /// A directory entry; `metadata.filename()` is its relative path.
152    Directory { metadata: FileMetadata },
153    /// Start of a file entry of `size` content bytes; [`ArchiveEvent::FileData`]
154    /// events follow, then [`ArchiveEvent::FileEnd`].
155    FileStart { metadata: FileMetadata, size: u64 },
156    /// A piece of the current file's content.
157    FileData(SecureBytes),
158    /// The current file's content is complete.
159    FileEnd,
160    /// The archive terminator was reached.
161    End,
162}
163
164enum State {
165    Magic,
166    EntryType,
167    EntryHeader { entry_type: u8 },
168    FileContent { remaining: u64 },
169    Finished,
170}
171
172/// Incremental archive parser: [`ArchiveParser::feed`] it byte pieces of any
173/// size, then drain [`ArchiveParser::next_event`] until it returns `None`
174/// (more input needed). Call [`ArchiveParser::finish`] after the last feed
175/// to catch truncated streams. Buffered bytes are zeroized on drop.
176pub struct ArchiveParser {
177    buf: Zeroizing<Vec<u8>>,
178    state: State,
179}
180
181impl ArchiveParser {
182    #[allow(clippy::new_without_default)]
183    pub fn new() -> Self {
184        Self {
185            buf: Zeroizing::new(Vec::new()),
186            state: State::Magic,
187        }
188    }
189
190    pub fn feed(&mut self, bytes: &[u8]) {
191        self.buf.extend_from_slice(bytes);
192    }
193
194    /// Returns the next event, or `None` when more input is needed.
195    pub fn next_event(&mut self) -> Result<Option<ArchiveEvent>, ArchiveError> {
196        match self.state {
197            State::Magic => {
198                if self.buf.len() < MAGIC.len() {
199                    return Ok(None);
200                }
201                if self.buf[..MAGIC.len()] != MAGIC {
202                    return Err(ArchiveError::InvalidData);
203                }
204                self.consume(MAGIC.len());
205                self.state = State::EntryType;
206                self.next_event()
207            }
208            State::EntryType => {
209                let Some(&entry_type) = self.buf.first() else {
210                    return Ok(None);
211                };
212                self.consume(1);
213                match entry_type {
214                    ENTRY_END => {
215                        self.state = State::Finished;
216                        if !self.buf.is_empty() {
217                            return Err(ArchiveError::TrailingData);
218                        }
219                        Ok(Some(ArchiveEvent::End))
220                    }
221                    ENTRY_FILE | ENTRY_DIR => {
222                        self.state = State::EntryHeader { entry_type };
223                        self.next_event()
224                    }
225                    _ => Err(ArchiveError::InvalidData),
226                }
227            }
228            State::EntryHeader { entry_type } => {
229                let Some((consumed, metadata, content_len)) =
230                    try_parse_entry_header(&self.buf, entry_type)?
231                else {
232                    return Ok(None);
233                };
234                self.consume(consumed);
235                if entry_type == ENTRY_DIR {
236                    self.state = State::EntryType;
237                    Ok(Some(ArchiveEvent::Directory { metadata }))
238                } else {
239                    self.state = State::FileContent {
240                        remaining: content_len,
241                    };
242                    Ok(Some(ArchiveEvent::FileStart {
243                        metadata,
244                        size: content_len,
245                    }))
246                }
247            }
248            State::FileContent { remaining } => {
249                if remaining == 0 {
250                    self.state = State::EntryType;
251                    return Ok(Some(ArchiveEvent::FileEnd));
252                }
253                if self.buf.is_empty() {
254                    return Ok(None);
255                }
256                let take = usize::try_from(remaining)
257                    .unwrap_or(usize::MAX)
258                    .min(self.buf.len());
259                let data = SecureBytes::new(self.buf[..take].to_vec());
260                self.consume(take);
261                self.state = State::FileContent {
262                    remaining: remaining - take as u64,
263                };
264                Ok(Some(ArchiveEvent::FileData(data)))
265            }
266            State::Finished => {
267                if self.buf.is_empty() {
268                    Ok(None)
269                } else {
270                    Err(ArchiveError::TrailingData)
271                }
272            }
273        }
274    }
275
276    /// Verifies the stream ended cleanly: terminator seen, no bytes left.
277    pub fn finish(&self) -> Result<(), ArchiveError> {
278        match self.state {
279            State::Finished if self.buf.is_empty() => Ok(()),
280            State::Finished => Err(ArchiveError::TrailingData),
281            _ => Err(ArchiveError::Truncated),
282        }
283    }
284
285    fn consume(&mut self, n: usize) {
286        self.buf.drain(..n);
287    }
288}
289
290/// Attempts to parse one entry header (without the type byte) from `buf`.
291/// Returns `None` when more bytes are needed.
292#[allow(clippy::type_complexity)]
293fn try_parse_entry_header(
294    buf: &[u8],
295    entry_type: u8,
296) -> Result<Option<(usize, FileMetadata, u64)>, ArchiveError> {
297    let Some(path_len_bytes) = buf.get(0..2) else {
298        return Ok(None);
299    };
300    let path_len = u16::from_le_bytes(path_len_bytes.try_into().unwrap()) as usize;
301    if path_len > MAX_PATH_LEN {
302        return Err(ArchiveError::InvalidPath);
303    }
304
305    let Some(&flags) = buf.get(2 + path_len) else {
306        return Ok(None);
307    };
308    if flags & !(FLAG_MTIME | FLAG_MODE) != 0 {
309        return Err(ArchiveError::InvalidData);
310    }
311
312    let mut total = 2 + path_len + 1;
313    if flags & FLAG_MTIME != 0 {
314        total += 12;
315    }
316    if flags & FLAG_MODE != 0 {
317        total += 4;
318    }
319    if entry_type == ENTRY_FILE {
320        total += 8;
321    }
322    if buf.len() < total {
323        return Ok(None);
324    }
325
326    let path = std::str::from_utf8(&buf[2..2 + path_len]).map_err(|_| ArchiveError::InvalidPath)?;
327    validate_path(path)?;
328
329    let mut offset = 2 + path_len + 1;
330    let mtime = if flags & FLAG_MTIME != 0 {
331        let secs = i64::from_le_bytes(buf[offset..offset + 8].try_into().unwrap());
332        let nanos = u32::from_le_bytes(buf[offset + 8..offset + 12].try_into().unwrap());
333        offset += 12;
334        if nanos >= 1_000_000_000 {
335            return Err(ArchiveError::InvalidData);
336        }
337        Some(parts_to_systemtime(secs, nanos).ok_or(ArchiveError::InvalidData)?)
338    } else {
339        None
340    };
341    let mode = if flags & FLAG_MODE != 0 {
342        let mode = u32::from_le_bytes(buf[offset..offset + 4].try_into().unwrap());
343        offset += 4;
344        Some(mode)
345    } else {
346        None
347    };
348    let content_len = if entry_type == ENTRY_FILE {
349        u64::from_le_bytes(buf[offset..offset + 8].try_into().unwrap())
350    } else {
351        0
352    };
353
354    let metadata = FileMetadata::new(SecureString::new(path.to_string()), mtime, mode);
355    Ok(Some((total, metadata, content_len)))
356}
357
358/// Splits a `SystemTime` into (seconds, nanoseconds) relative to the Unix
359/// epoch, with pre-epoch times as negative seconds and nanos in `[0, 1e9)`.
360/// Total for any `SystemTime`: the seconds saturate at the i64 range
361/// (hundreds of billions of years out), so extreme timestamps can never
362/// overflow — found by fuzzing with `mtime_secs = i64::MIN`.
363fn systemtime_to_parts(t: SystemTime) -> (i64, u32) {
364    let (secs, nanos): (i128, u32) = match t.duration_since(UNIX_EPOCH) {
365        Ok(d) => (d.as_secs().into(), d.subsec_nanos()),
366        Err(e) => {
367            let d = e.duration();
368            let (secs, nanos) = (i128::from(d.as_secs()), d.subsec_nanos());
369            if nanos == 0 {
370                (-secs, 0)
371            } else {
372                (-(secs + 1), 1_000_000_000 - nanos)
373            }
374        }
375    };
376    (secs.clamp(i64::MIN.into(), i64::MAX.into()) as i64, nanos)
377}
378
379fn parts_to_systemtime(secs: i64, nanos: u32) -> Option<SystemTime> {
380    if secs >= 0 {
381        UNIX_EPOCH.checked_add(Duration::new(secs as u64, nanos))
382    } else if nanos == 0 {
383        UNIX_EPOCH.checked_sub(Duration::from_secs(secs.unsigned_abs()))
384    } else {
385        UNIX_EPOCH.checked_sub(Duration::new(
386            (secs + 1).unsigned_abs(),
387            1_000_000_000 - nanos,
388        ))
389    }
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395
396    fn meta(path: &str) -> FileMetadata {
397        FileMetadata::new(
398            SecureString::new(path.to_string()),
399            Some(UNIX_EPOCH + Duration::new(1_700_000_000, 42)),
400            Some(0o644),
401        )
402    }
403
404    /// Encodes a small archive: a directory, a file with content, an empty
405    /// file, and the terminator.
406    fn sample_archive() -> Vec<u8> {
407        let mut bytes = MAGIC.to_vec();
408        bytes.extend_from_slice(&encode_directory(&meta("sub")).unwrap());
409        bytes.extend_from_slice(&encode_file(&meta("sub/a.txt"), 11).unwrap());
410        bytes.extend_from_slice(b"hello world");
411        bytes.extend_from_slice(&encode_file(&meta("empty.bin"), 0).unwrap());
412        bytes.extend_from_slice(&encode_end());
413        bytes
414    }
415
416    /// Feeds the archive in pieces of `piece_len` and collects all events.
417    fn parse_in_pieces(bytes: &[u8], piece_len: usize) -> Result<Vec<String>, ArchiveError> {
418        let mut parser = ArchiveParser::new();
419        let mut events = Vec::new();
420        let mut content = Vec::new();
421        for piece in bytes.chunks(piece_len.max(1)) {
422            parser.feed(piece);
423            while let Some(event) = parser.next_event()? {
424                match event {
425                    ArchiveEvent::Directory { metadata } => {
426                        events.push(format!("dir:{}", metadata.filename().as_str()));
427                    }
428                    ArchiveEvent::FileStart { metadata, size } => {
429                        content.clear();
430                        events.push(format!("file:{}:{}", metadata.filename().as_str(), size));
431                    }
432                    ArchiveEvent::FileData(data) => content.extend_from_slice(data.as_slice()),
433                    ArchiveEvent::FileEnd => {
434                        events.push(format!("data:{}", String::from_utf8_lossy(&content)));
435                    }
436                    ArchiveEvent::End => events.push("end".to_string()),
437                }
438            }
439        }
440        parser.finish()?;
441        Ok(events)
442    }
443
444    #[test]
445    fn round_trip_all_piece_sizes() {
446        let bytes = sample_archive();
447        // Byte-at-a-time up through whole-buffer feeds must all parse the same.
448        for piece_len in [1, 2, 3, 7, 16, bytes.len()] {
449            let events = parse_in_pieces(&bytes, piece_len).unwrap();
450            assert_eq!(
451                events,
452                vec![
453                    "dir:sub",
454                    "file:sub/a.txt:11",
455                    "data:hello world",
456                    "file:empty.bin:0",
457                    "data:",
458                    "end",
459                ],
460                "piece_len {piece_len}"
461            );
462        }
463    }
464
465    #[test]
466    fn entry_metadata_round_trips() {
467        let mut bytes = MAGIC.to_vec();
468        bytes.extend_from_slice(&encode_file(&meta("f"), 0).unwrap());
469        bytes.extend_from_slice(&encode_end());
470
471        let mut parser = ArchiveParser::new();
472        parser.feed(&bytes);
473        let Some(ArchiveEvent::FileStart { metadata, .. }) = parser.next_event().unwrap() else {
474            panic!("expected FileStart");
475        };
476        assert_eq!(metadata.mtime(), meta("f").mtime());
477        assert_eq!(metadata.mode(), Some(0o644));
478    }
479
480    #[test]
481    fn empty_archive_round_trips() {
482        let mut bytes = MAGIC.to_vec();
483        bytes.extend_from_slice(&encode_end());
484        assert_eq!(parse_in_pieces(&bytes, 1).unwrap(), vec!["end"]);
485    }
486
487    #[test]
488    fn truncated_stream_is_detected() {
489        let bytes = sample_archive();
490        for len in 0..bytes.len() - 1 {
491            assert!(
492                parse_in_pieces(&bytes[..len], 64).is_err(),
493                "accepted truncation at {len}"
494            );
495        }
496    }
497
498    #[test]
499    fn trailing_data_is_detected() {
500        let mut bytes = sample_archive();
501        bytes.push(0);
502        assert!(matches!(
503            parse_in_pieces(&bytes, 64),
504            Err(ArchiveError::TrailingData)
505        ));
506    }
507
508    #[test]
509    fn wrong_magic_is_detected() {
510        let mut bytes = sample_archive();
511        bytes[0] ^= 1;
512        assert!(matches!(
513            parse_in_pieces(&bytes, 64),
514            Err(ArchiveError::InvalidData)
515        ));
516    }
517
518    #[test]
519    fn unknown_entry_type_is_detected() {
520        let mut bytes = MAGIC.to_vec();
521        bytes.push(9);
522        assert!(matches!(
523            parse_in_pieces(&bytes, 64),
524            Err(ArchiveError::InvalidData)
525        ));
526    }
527
528    #[test]
529    fn evil_paths_are_rejected_on_encode_and_parse() {
530        for path in [
531            "",
532            "..",
533            "../etc/passwd",
534            "a/../b",
535            "/abs",
536            "a//b",
537            "a/./b",
538            "a\\b",
539            "a/",
540        ] {
541            assert!(
542                encode_file(
543                    &FileMetadata::new(SecureString::new(path.to_string()), None, None),
544                    0
545                )
546                .is_err(),
547                "encode accepted {path:?}"
548            );
549
550            // Hand-craft the same path into a stream to test the parse side.
551            let mut bytes = MAGIC.to_vec();
552            bytes.push(1);
553            bytes.extend_from_slice(&(path.len() as u16).to_le_bytes());
554            bytes.extend_from_slice(path.as_bytes());
555            bytes.push(0); // flags
556            bytes.extend_from_slice(&0u64.to_le_bytes());
557            bytes.extend_from_slice(&encode_end());
558            assert!(
559                parse_in_pieces(&bytes, 64).is_err(),
560                "parse accepted {path:?}"
561            );
562        }
563    }
564
565    #[test]
566    fn valid_nested_path_is_accepted() {
567        assert!(validate_path("a/b/c.txt").is_ok());
568        assert!(validate_path("single").is_ok());
569    }
570}
shadow_crypt_core/archive.rs

shadow_crypt_core/
archive.rs
