lit/storage/
pack.rs

1//! Pack file format — many objects in one file, with a hash→offset index.
2//!
3//! A pack is written by `lit gc` and read back through [`ObjectStore`], which
4//! consults packs whenever a hash has no loose object on disk. Both sides live
5//! here so the format is defined in one place: a reader and a writer that drift
6//! apart is how packed objects become unreadable.
7//!
8//! Layout, all integers big-endian:
9//!
10//! ```text
11//! pack:  "LITP" version:u32 count:u32
12//!        then per object: type:u8 uncompressed_len:u64 compressed_len:u64 zlib-data
13//! index: "LITI" version:u32 count:u32
14//!        then per object, sorted by hash: hash_len:u32 hash offset:u64 crc32:u32
15//! ```
16//!
17//! [`ObjectStore`]: crate::storage::ObjectStore
18
19use crate::core::{Object, ObjectHash};
20use crate::crypto::encryption::EncryptionManager;
21use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
22use crc32fast::Hasher as Crc32Hasher;
23use flate2::read::ZlibDecoder;
24use flate2::write::ZlibEncoder;
25use flate2::Compression;
26use std::collections::HashMap;
27use std::fs;
28use std::io::{Cursor, Read, Write};
29use std::path::{Path, PathBuf};
30
31/// Magic header for Lit pack files
32pub const PACK_MAGIC: &[u8; 4] = b"LITP";
33/// Pack version
34pub const PACK_VERSION: u32 = 1;
35
36/// Index magic header
37pub const INDEX_MAGIC: &[u8; 4] = b"LITI";
38/// Index version
39pub const INDEX_VERSION: u32 = 1;
40
41/// Pack index entry - maps hash to offset in pack
42#[derive(Debug, Clone)]
43pub struct PackIndexEntry {
44    pub hash: ObjectHash,
45    pub offset: u64,
46    pub crc32: u32,
47}
48
49/// The directory holding a repository's packs.
50pub fn packs_dir(repo_root: &Path) -> PathBuf {
51    repo_root.join(".lit").join("packs")
52}
53
54/// Write a pack file from a set of objects.
55///
56/// `encryption` must be the same manager the loose objects were written
57/// through. A packed object gets the identical compress-then-encrypt treatment,
58/// so packing an encrypted repository cannot quietly put its contents on disk
59/// in the clear. When encryption is disabled the manager passes bytes through
60/// untouched and the pack is plain zlib.
61pub fn write_pack(
62    objects: &[(ObjectHash, Object)],
63    pack_path: &Path,
64    encryption: &EncryptionManager,
65) -> Result<Vec<PackIndexEntry>, crate::errors::LitError> {
66    let mut buf: Vec<u8> = Vec::new();
67    let mut index_entries = Vec::new();
68
69    // Header: magic + version + count
70    buf.extend_from_slice(PACK_MAGIC);
71    buf.write_u32::<BigEndian>(PACK_VERSION)
72        .map_err(|e| format!("Write error: {}", e))?;
73    buf.write_u32::<BigEndian>(objects.len() as u32)
74        .map_err(|e| format!("Write error: {}", e))?;
75
76    for (hash, obj) in objects {
77        let offset = buf.len() as u64;
78
79        let type_byte = match obj {
80            Object::Blob(_) => 1u8,
81            Object::Tree(_) => 2u8,
82            Object::Commit(_) => 3u8,
83            Object::Tag(_) => 4u8,
84        };
85
86        let raw = obj.to_bytes();
87        let uncompressed_size = raw.len() as u64;
88
89        // Compress data
90        let mut encoder = ZlibEncoder::new(Vec::new(), Compression::best());
91        encoder
92            .write_all(&raw)
93            .map_err(|e| format!("Compress error: {}", e))?;
94        let compressed = encoder
95            .finish()
96            .map_err(|e| format!("Compress finish error: {}", e))?;
97
98        // Same treatment the loose objects get: compress, then encrypt.
99        let stored = encryption.encrypt(&compressed)?;
100
101        // CRC32 covers what is actually on disk
102        let mut crc32 = Crc32Hasher::new();
103        crc32.update(&stored);
104        let crc_val = crc32.finalize();
105
106        // Write entry header: type(1) + uncompressed_size(8) + stored_size(8)
107        buf.push(type_byte);
108        buf.write_u64::<BigEndian>(uncompressed_size)
109            .map_err(|e| format!("Write error: {}", e))?;
110        buf.write_u64::<BigEndian>(stored.len() as u64)
111            .map_err(|e| format!("Write error: {}", e))?;
112        buf.extend_from_slice(&stored);
113
114        index_entries.push(PackIndexEntry {
115            hash: hash.clone(),
116            offset,
117            crc32: crc_val,
118        });
119    }
120
121    fs::write(pack_path, &buf).map_err(|e| format!("Failed to write pack: {}", e))?;
122
123    Ok(index_entries)
124}
125
126/// Write a pack index file
127pub fn write_pack_index(
128    entries: &[PackIndexEntry],
129    index_path: &Path,
130) -> Result<(), crate::errors::LitError> {
131    let mut buf: Vec<u8> = Vec::new();
132
133    // Header
134    buf.extend_from_slice(INDEX_MAGIC);
135    buf.write_u32::<BigEndian>(INDEX_VERSION)
136        .map_err(|e| format!("Write error: {}", e))?;
137    buf.write_u32::<BigEndian>(entries.len() as u32)
138        .map_err(|e| format!("Write error: {}", e))?;
139
140    // Write sorted entries: hash_len(4) + hash + offset(8) + crc32(4)
141    let mut sorted = entries.to_vec();
142    sorted.sort_by(|a, b| a.hash.as_str().cmp(b.hash.as_str()));
143
144    for entry in &sorted {
145        let hash_bytes = entry.hash.as_str().as_bytes();
146        buf.write_u32::<BigEndian>(hash_bytes.len() as u32)
147            .map_err(|e| format!("Write error: {}", e))?;
148        buf.extend_from_slice(hash_bytes);
149        buf.write_u64::<BigEndian>(entry.offset)
150            .map_err(|e| format!("Write error: {}", e))?;
151        buf.write_u32::<BigEndian>(entry.crc32)
152            .map_err(|e| format!("Write error: {}", e))?;
153    }
154
155    fs::write(index_path, &buf).map_err(|e| format!("Failed to write index: {}", e))?;
156    Ok(())
157}
158
159/// Read a single object from a pack file by offset
160pub fn read_pack_object(
161    pack_path: &Path,
162    offset: u64,
163    encryption: &EncryptionManager,
164) -> Result<Object, crate::errors::LitError> {
165    let pack_data = fs::read(pack_path).map_err(|e| format!("Failed to read pack: {}", e))?;
166    let mut cursor = Cursor::new(&pack_data);
167    cursor.set_position(offset);
168
169    // Read type
170    let _type_byte = cursor.read_u8().map_err(|e| format!("Read error: {}", e))?;
171
172    // Read sizes
173    let _uncompressed_size = cursor
174        .read_u64::<BigEndian>()
175        .map_err(|e| format!("Read error: {}", e))?;
176    let compressed_size = cursor
177        .read_u64::<BigEndian>()
178        .map_err(|e| format!("Read error: {}", e))?;
179
180    // Read compressed data
181    let pos = cursor.position() as usize;
182    let end = pos
183        .checked_add(compressed_size as usize)
184        .ok_or("Pack entry length overflows")?;
185    if end > pack_data.len() {
186        return Err("Pack data truncated".into());
187    }
188    let stored = &pack_data[pos..end];
189
190    // Undo the write path: decrypt, then decompress.
191    let compressed = encryption.decrypt(stored)?;
192
193    // Decompress
194    let mut decoder = ZlibDecoder::new(&compressed[..]);
195    let mut raw = Vec::new();
196    decoder
197        .read_to_end(&mut raw)
198        .map_err(|e| format!("Decompress error: {}", e))?;
199
200    Object::from_bytes(&raw).map_err(Into::into)
201}
202
203/// Load a pack index and build a hash→(pack_path, offset) map
204pub fn load_pack_index(index_path: &Path) -> Result<HashMap<String, (PathBuf, u64)>, String> {
205    let data = fs::read(index_path).map_err(|e| format!("Failed to read index: {}", e))?;
206    let mut cursor = Cursor::new(&data);
207
208    // Verify header
209    let mut magic = [0u8; 4];
210    cursor
211        .read_exact(&mut magic)
212        .map_err(|e| format!("Read error: {}", e))?;
213    if &magic != INDEX_MAGIC {
214        return Err("Invalid pack index magic".into());
215    }
216
217    let _version = cursor
218        .read_u32::<BigEndian>()
219        .map_err(|e| format!("Read error: {}", e))?;
220    let count = cursor
221        .read_u32::<BigEndian>()
222        .map_err(|e| format!("Read error: {}", e))?;
223
224    let pack_path = index_path.with_extension("pack");
225    let mut map = HashMap::new();
226
227    for _ in 0..count {
228        let hash_len = cursor
229            .read_u32::<BigEndian>()
230            .map_err(|e| format!("Read error: {}", e))? as usize;
231
232        let pos = cursor.position() as usize;
233        if pos + hash_len > data.len() {
234            return Err("Index data truncated".into());
235        }
236        let hash_str = String::from_utf8(data[pos..pos + hash_len].to_vec())
237            .map_err(|e| format!("Invalid hash UTF-8: {}", e))?;
238        cursor.set_position((pos + hash_len) as u64);
239
240        let offset = cursor
241            .read_u64::<BigEndian>()
242            .map_err(|e| format!("Read error: {}", e))?;
243        let _crc32 = cursor
244            .read_u32::<BigEndian>()
245            .map_err(|e| format!("Read error: {}", e))?;
246
247        map.insert(hash_str, (pack_path.clone(), offset));
248    }
249
250    Ok(map)
251}
252
253/// Merge every pack index in `packs_dir` into one hash→(pack, offset) map.
254///
255/// A pack whose index will not parse is skipped rather than failing the whole
256/// lookup: one damaged pack should not make the objects in the others
257/// unreachable. Missing directory means no packs, which is not an error.
258pub fn load_all(packs_dir: &Path) -> HashMap<String, (PathBuf, u64)> {
259    let mut map = HashMap::new();
260
261    let entries = match fs::read_dir(packs_dir) {
262        Ok(entries) => entries,
263        Err(_) => return map,
264    };
265
266    for entry in entries.flatten() {
267        let path = entry.path();
268        if path.extension().and_then(|e| e.to_str()) != Some("idx") {
269            continue;
270        }
271        match load_pack_index(&path) {
272            Ok(entries) => map.extend(entries),
273            Err(e) => eprintln!("Warning: ignoring unreadable pack index {:?}: {}", path, e),
274        }
275    }
276
277    map
278}
lit/storage/pack.rs

lit/storage/
pack.rs
