git_internal/internal/pack/
stats.rs1use std::{
2 collections::HashSet,
3 fmt,
4 fs::File,
5 io::{self, BufRead, BufReader, ErrorKind, Read},
6 path::Path,
7};
8
9use flate2::bufread::ZlibDecoder;
10
11use crate::{
12 errors::GitError,
13 hash::{ObjectHash, get_hash_kind},
14 internal::pack::{Pack, utils, wrapper::Wrapper},
15 utils::{CountingReader, HashAlgorithm},
16};
17
18#[derive(Debug, Default, Clone, PartialEq, Eq)]
19pub struct PackStats {
20 pub total: usize,
21 pub commits: usize,
22 pub trees: usize,
23 pub blobs: usize,
24 pub tags: usize,
25 pub deltas: usize,
26}
27
28struct PackScan {
29 stats: PackStats,
30 ref_delta_bases: HashSet<ObjectHash>,
31 pack_hash: ObjectHash,
32}
33
34struct PackIndexHashes {
35 objects: HashSet<ObjectHash>,
36 pack_hash: ObjectHash,
37}
38
39struct HashingReader<R> {
40 inner: R,
41 hash: HashAlgorithm,
42}
43
44impl<R> HashingReader<R> {
45 fn new(inner: R) -> Self {
46 Self {
47 inner,
48 hash: HashAlgorithm::new(),
49 }
50 }
51
52 fn current_hash(&self) -> Result<ObjectHash, GitError> {
53 ObjectHash::from_bytes(&self.hash.clone().finalize())
54 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))
55 }
56}
57
58impl<R: Read> HashingReader<R> {
59 fn read_without_hash(&mut self, buf: &mut [u8]) -> io::Result<usize> {
60 self.inner.read(buf)
61 }
62
63 fn read_exact_without_hash(&mut self, buf: &mut [u8]) -> io::Result<()> {
64 self.inner.read_exact(buf)
65 }
66}
67
68impl<R: Read> Read for HashingReader<R> {
69 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
70 let n = self.inner.read(buf)?;
71 self.hash.update(&buf[..n]);
72 Ok(n)
73 }
74}
75
76impl fmt::Display for PackStats {
77 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
78 write!(
79 f,
80 "PackStats {{ total: {}, commits: {}, trees: {}, blobs: {}, tags: {}, deltas: {} }}",
81 self.total, self.commits, self.trees, self.blobs, self.tags, self.deltas
82 )
83 }
84}
85
86impl PackStats {
87 pub fn analyze<P: AsRef<Path>>(pack_path: P) -> Result<PackStats, GitError> {
88 let pack_path = pack_path.as_ref();
89 if !pack_path.exists() {
90 return Err(GitError::InvalidPackFile(format!(
91 "Pack file not found: {}",
92 pack_path.display()
93 )));
94 }
95
96 let f = File::open(pack_path)
97 .map_err(|e| GitError::InvalidPackFile(format!("Failed to open pack file: {e}")))?;
98 let scan = Self::scan(BufReader::new(f))?;
99 if !scan.ref_delta_bases.is_empty() {
100 let index_hashes = Self::read_pack_index_hashes(pack_path)?.ok_or_else(|| {
101 GitError::InvalidPackFile(
102 "Pack index is required to verify ref-delta bases".to_string(),
103 )
104 })?;
105 if index_hashes.pack_hash != scan.pack_hash {
106 return Err(GitError::InvalidPackFile(format!(
107 "Pack index hash {} does not match pack trailer hash {}",
108 index_hashes.pack_hash, scan.pack_hash
109 )));
110 }
111 if let Some(base_hash) = scan
112 .ref_delta_bases
113 .iter()
114 .find(|base_hash| !index_hashes.objects.contains(*base_hash))
115 {
116 return Err(GitError::InvalidPackFile(format!(
117 "Ref-delta base {base_hash} is not present in the pack index"
118 )));
119 }
120 }
121
122 Ok(scan.stats)
123 }
124
125 pub fn validate_header<P: AsRef<Path>>(pack_path: P) -> Result<u32, GitError> {
126 let pack_path = pack_path.as_ref();
127 if !pack_path.exists() {
128 return Err(GitError::InvalidPackFile(format!(
129 "Pack file not found: {}",
130 pack_path.display()
131 )));
132 }
133
134 let f = File::open(pack_path)
135 .map_err(|e| GitError::InvalidPackFile(format!("Failed to open pack file: {e}")))?;
136 let mut reader = BufReader::new(f);
137
138 let (count, _) = Pack::check_header(&mut reader)?;
139 Ok(count)
140 }
141
142 fn scan(reader: impl BufRead) -> Result<PackScan, GitError> {
143 let mut reader = Wrapper::new(reader);
144 let (object_num, header_data) = Pack::check_header(&mut reader)?;
145 let mut stats = PackStats {
146 total: object_num as usize,
147 ..Default::default()
148 };
149 let first_object_offset = header_data.len();
150 let mut offset = first_object_offset;
151 let mut object_starts = HashSet::new();
152 let mut ref_delta_bases = HashSet::new();
153
154 for _ in 0..object_num {
155 let object_start = offset;
156 let (type_bits, size) = utils::read_type_and_varint_size(&mut reader, &mut offset)
157 .map_err(|e| {
158 GitError::InvalidPackFile(format!("Read error at offset {offset}: {e}"))
159 })?;
160
161 stats.count_type_bits(type_bits, offset)?;
162
163 match type_bits {
164 1..=4 => drain_zlib(&mut reader, &mut offset, size)?,
165 5 | 6 => {
166 let (delta_offset, consumed) = utils::read_offset_encoding(&mut reader)
167 .map_err(|e| {
168 GitError::InvalidPackFile(format!(
169 "Read offset encoding error at offset {offset}: {e}"
170 ))
171 })?;
172 let delta_offset = usize::try_from(delta_offset).map_err(|_| {
173 GitError::InvalidPackFile(format!(
174 "Offset delta at {object_start} exceeds platform limits"
175 ))
176 })?;
177 let base_offset = object_start.checked_sub(delta_offset).ok_or_else(|| {
178 GitError::InvalidPackFile(format!(
179 "Offset delta at {object_start} points before pack data"
180 ))
181 })?;
182 if delta_offset == 0
183 || base_offset < first_object_offset
184 || !object_starts.contains(&base_offset)
185 {
186 return Err(GitError::InvalidPackFile(format!(
187 "Offset delta at {object_start} does not reference an earlier object"
188 )));
189 }
190 add_to_offset(&mut offset, consumed)?;
191 drain_zlib(&mut reader, &mut offset, size)?;
192 }
193 7 => {
194 let base_hash = ObjectHash::from_stream(&mut reader).map_err(|e| {
195 GitError::InvalidPackFile(format!(
196 "Read hash error at offset {offset}: {e}"
197 ))
198 })?;
199 add_to_offset(&mut offset, base_hash.size())?;
200 ref_delta_bases.insert(base_hash);
201 drain_zlib(&mut reader, &mut offset, size)?;
202 }
203 _ => unreachable!(),
204 }
205 object_starts.insert(object_start);
206 }
207
208 let computed_hash = reader.final_hash();
209 let trailer = ObjectHash::from_stream(&mut reader).map_err(|e| {
210 GitError::InvalidPackFile(format!("Failed to read trailer hash: {e:?}"))
211 })?;
212 if computed_hash != trailer {
213 return Err(GitError::InvalidPackFile(format!(
214 "Pack trailer mismatch: computed {computed_hash}, stored {trailer}"
215 )));
216 }
217 if !utils::is_eof(&mut reader) {
218 return Err(GitError::InvalidPackFile(
219 "Pack has trailing data after trailer".to_string(),
220 ));
221 }
222
223 Ok(PackScan {
224 stats,
225 ref_delta_bases,
226 pack_hash: trailer,
227 })
228 }
229
230 fn read_pack_index_hashes(pack_path: &Path) -> Result<Option<PackIndexHashes>, GitError> {
231 let idx_path = pack_path.with_extension("idx");
232 let idx_file = match File::open(&idx_path) {
233 Ok(file) => file,
234 Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None),
235 Err(e) => {
236 return Err(GitError::InvalidPackFile(format!(
237 "Failed to open pack index {}: {e}",
238 idx_path.display()
239 )));
240 }
241 };
242 let mut reader = HashingReader::new(BufReader::new(idx_file));
243
244 let magic = read_be_u32(&mut reader)?;
245 let version = read_be_u32(&mut reader)?;
246 if magic != 0xff74_4f63 || version != 2 {
247 return Err(GitError::InvalidPackFile(
248 "Only pack index v2 is supported for ref-delta validation".to_string(),
249 ));
250 }
251
252 let mut object_num = 0usize;
253 for _ in 0..256 {
254 object_num = read_be_u32(&mut reader)? as usize;
255 }
256
257 let hash_size = get_hash_kind().size();
258 let mut objects = HashSet::new();
259 let mut hash_buf = vec![0; hash_size];
260 for _ in 0..object_num {
261 reader
262 .read_exact(&mut hash_buf)
263 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
264 let hash = ObjectHash::from_bytes(&hash_buf)
265 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
266 objects.insert(hash);
267 }
268
269 let crc_bytes = checked_mul(object_num, 4, "Pack index is too large")?;
270 discard_exact(&mut reader, crc_bytes)?;
271
272 let mut large_offset_count = 0usize;
273 for _ in 0..object_num {
274 let offset = read_be_u32(&mut reader)?;
275 if offset & 0x8000_0000 != 0 {
276 large_offset_count = large_offset_count.checked_add(1).ok_or_else(|| {
277 GitError::InvalidPackFile("Pack index is too large".to_string())
278 })?;
279 }
280 }
281 let large_offset_bytes = checked_mul(large_offset_count, 8, "Pack index is too large")?;
282 discard_exact(&mut reader, large_offset_bytes)?;
283
284 let pack_hash = ObjectHash::from_stream(&mut reader)
285 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
286 let expected_idx_hash = reader.current_hash()?;
287 let mut idx_hash_buf = vec![0; hash_size];
288 reader
289 .read_exact_without_hash(&mut idx_hash_buf)
290 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
291 let idx_hash = ObjectHash::from_bytes(&idx_hash_buf)
292 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
293 if idx_hash != expected_idx_hash {
294 return Err(GitError::InvalidPackFile(format!(
295 "Pack index checksum {idx_hash} does not match calculated checksum {expected_idx_hash}"
296 )));
297 }
298 let mut trailing = [0; 1];
299 if reader
300 .read_without_hash(&mut trailing)
301 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?
302 != 0
303 {
304 return Err(GitError::InvalidPackFile(
305 "Pack index has trailing data after checksum".to_string(),
306 ));
307 }
308
309 Ok(Some(PackIndexHashes { objects, pack_hash }))
310 }
311
312 fn count_type_bits(&mut self, type_bits: u8, offset: usize) -> Result<(), GitError> {
313 match type_bits {
314 1 => {
315 self.commits += 1;
316 }
317 2 => {
318 self.trees += 1;
319 }
320 3 => {
321 self.blobs += 1;
322 }
323 4 => {
324 self.tags += 1;
325 }
326 5..=7 => {
327 self.deltas += 1;
328 }
329 _ => {
330 return Err(GitError::InvalidObjectType(format!(
331 "Unknown pack type bits: {type_bits} at offset {offset}"
332 )));
333 }
334 }
335 Ok(())
336 }
337}
338
339fn drain_zlib(
340 reader: &mut impl BufRead,
341 offset: &mut usize,
342 expected_size: usize,
343) -> Result<(), GitError> {
344 let mut counting_reader = CountingReader::new(reader);
345 let mut deflate = ZlibDecoder::new(&mut counting_reader);
346 let mut remaining = expected_size;
347 let mut scratch = [0; 8192];
348
349 while remaining > 0 {
350 let chunk_len = remaining.min(scratch.len());
351 let bytes = deflate
352 .read(&mut scratch[..chunk_len])
353 .map_err(|e| GitError::InvalidPackFile(format!("Decompression error: {e}")))?;
354 if bytes == 0 {
355 return Err(GitError::InvalidPackFile(format!(
356 "The object size is smaller than the expected size {expected_size}"
357 )));
358 }
359 remaining -= bytes;
360 }
361
362 let mut extra = [0; 1];
363 let extra_bytes = deflate
364 .read(&mut extra)
365 .map_err(|e| GitError::InvalidPackFile(format!("Decompression error: {e}")))?;
366 if extra_bytes != 0 {
367 return Err(GitError::InvalidPackFile(format!(
368 "The object size exceeds the expected size {expected_size}"
369 )));
370 }
371
372 let consumed = usize::try_from(counting_reader.bytes_read).map_err(|_| {
373 GitError::InvalidPackFile("Compressed object size exceeds platform limits".to_string())
374 })?;
375 add_to_offset(offset, consumed)
376}
377
378fn add_to_offset(offset: &mut usize, consumed: usize) -> Result<(), GitError> {
379 *offset = offset
380 .checked_add(consumed)
381 .ok_or_else(|| GitError::InvalidPackFile("Pack offset overflow".to_string()))?;
382 Ok(())
383}
384
385fn read_be_u32(reader: &mut impl Read) -> Result<u32, GitError> {
386 let mut buf = [0; 4];
387 reader
388 .read_exact(&mut buf)
389 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
390 Ok(u32::from_be_bytes(buf))
391}
392
393fn discard_exact(reader: &mut impl Read, mut len: usize) -> Result<(), GitError> {
394 let mut scratch = [0; 8192];
395 while len != 0 {
396 let chunk_len = len.min(scratch.len());
397 reader
398 .read_exact(&mut scratch[..chunk_len])
399 .map_err(|e| GitError::InvalidPackFile(format!("Read index error: {e}")))?;
400 len -= chunk_len;
401 }
402 Ok(())
403}
404
405fn checked_mul(lhs: usize, rhs: usize, message: &str) -> Result<usize, GitError> {
406 lhs.checked_mul(rhs)
407 .ok_or_else(|| GitError::InvalidPackFile(message.to_string()))
408}
409
410#[cfg(test)]
411mod tests {
412 use super::*;
413 use crate::{
414 hash::{HashKind, set_hash_kind_for_test},
415 internal::pack::test_pack_download::download_pack_file,
416 };
417
418 #[test]
419 fn test_analyze_small_pack_sha1() {
420 let _guard = set_hash_kind_for_test(HashKind::Sha1);
421 let (pack_path, _dl_guard) = download_pack_file("small-sha1.pack");
422 let stats = PackStats::analyze(pack_path).expect("Failed to analyze");
423
424 assert!(stats.total > 0);
425 assert_eq!(
426 stats.total,
427 stats.commits + stats.trees + stats.blobs + stats.tags + stats.deltas
428 );
429 }
430
431 #[test]
432 fn test_analyze_small_pack_sha256() {
433 let _guard = set_hash_kind_for_test(HashKind::Sha256);
434 let (pack_path, _dl_guard) = download_pack_file("small-sha256.pack");
435 let stats = PackStats::analyze(pack_path).expect("Failed to analyze");
436
437 assert!(stats.total > 0);
438 assert_eq!(
439 stats.total,
440 stats.commits + stats.trees + stats.blobs + stats.tags + stats.deltas
441 );
442 }
443
444 #[test]
445 fn test_analyze_delta_pack_sha1() {
446 let _guard = set_hash_kind_for_test(HashKind::Sha1);
447 let (pack_path, _dl_guard) = download_pack_file("ref-delta-sha1.pack");
448 let stats = PackStats::analyze(pack_path).expect("Failed to analyze");
449
450 assert!(stats.total > 0);
451
452 assert_eq!(
453 stats.total,
454 stats.commits + stats.trees + stats.blobs + stats.tags + stats.deltas
455 );
456 }
457
458 #[test]
459 fn test_analyze_delta_pack_sha256() {
460 let _guard = set_hash_kind_for_test(HashKind::Sha256);
461 let (pack_path, _dl_guard) = download_pack_file("ref-delta-sha256.pack");
462 let stats = PackStats::analyze(pack_path).expect("Failed to analyze");
463
464 assert!(stats.total > 0);
465 assert_eq!(
466 stats.total,
467 stats.commits + stats.trees + stats.blobs + stats.tags + stats.deltas
468 );
469 }
470
471 #[test]
472 fn test_nonexistent_file() {
473 let result = PackStats::analyze("tests/data/packs/nonexistent.pack");
474 assert!(result.is_err());
475 }
476
477 #[test]
478 fn test_invalid_pack_file() {
479 use std::io::Write;
480
481 use tempfile::NamedTempFile;
482
483 let mut temp = NamedTempFile::new().expect("create temp file");
484 temp.write_all(b"XXXX").expect("write temp file");
485 temp.flush().expect("flush temp file");
486
487 let result = PackStats::analyze(temp.path());
488 assert!(result.is_err());
489 }
490
491 #[test]
492 fn test_analyze_rejects_trailing_data() {
493 use std::{fs, io::Write};
494
495 use tempfile::NamedTempFile;
496
497 let _guard = set_hash_kind_for_test(HashKind::Sha1);
498 let (pack_path, _dl_guard) = download_pack_file("small-sha1.pack");
499 let mut bytes = fs::read(pack_path).expect("read pack fixture");
500 bytes.push(0);
501
502 let mut temp = NamedTempFile::new().expect("create temp file");
503 temp.write_all(&bytes).expect("write temp file");
504 temp.flush().expect("flush temp file");
505
506 let result = PackStats::analyze(temp.path());
507 assert!(result.is_err());
508 }
509
510 #[test]
511 fn test_analyze_huge_object_count_does_not_preallocate() {
512 use std::io::Write;
513
514 use tempfile::NamedTempFile;
515
516 let _guard = set_hash_kind_for_test(HashKind::Sha1);
517 let mut bytes = Vec::new();
518 bytes.extend_from_slice(b"PACK");
519 bytes.extend_from_slice(&2u32.to_be_bytes());
520 bytes.extend_from_slice(&u32::MAX.to_be_bytes());
521
522 let mut temp = NamedTempFile::new().expect("create temp file");
523 temp.write_all(&bytes).expect("write temp file");
524 temp.flush().expect("flush temp file");
525
526 let result = PackStats::analyze(temp.path());
527 assert!(result.is_err());
528 }
529
530 #[test]
531 fn test_analyze_ref_delta_requires_index() {
532 use std::io::Write;
533
534 use flate2::{Compression, write::ZlibEncoder};
535 use tempfile::NamedTempFile;
536
537 let _guard = set_hash_kind_for_test(HashKind::Sha1);
538 let mut deflate = ZlibEncoder::new(Vec::new(), Compression::default());
539 deflate.write_all(&[]).expect("write zlib payload");
540 let compressed_delta = deflate.finish().expect("finish zlib payload");
541
542 let mut bytes = Vec::new();
543 bytes.extend_from_slice(b"PACK");
544 bytes.extend_from_slice(&2u32.to_be_bytes());
545 bytes.extend_from_slice(&1u32.to_be_bytes());
546 write_pack_object_header(&mut bytes, 7, 0);
547 bytes.extend(std::iter::repeat_n(0, get_hash_kind().size()));
548 bytes.extend_from_slice(&compressed_delta);
549 append_pack_trailer(&mut bytes);
550
551 let mut temp = NamedTempFile::new().expect("create temp file");
552 temp.write_all(&bytes).expect("write temp file");
553 temp.flush().expect("flush temp file");
554
555 let result = PackStats::analyze(temp.path());
556 assert!(format!("{result:?}").contains("Pack index is required"));
557 }
558
559 #[test]
560 fn test_validate_header() {
561 let _guard = set_hash_kind_for_test(HashKind::Sha1);
562 let (pack_path, _dl_guard) = download_pack_file("small-sha1.pack");
563 let result = PackStats::validate_header(pack_path);
564 assert!(result.is_ok());
565 assert!(result.unwrap() > 0);
566 }
567
568 #[test]
569 fn test_validate_header_nonexistent() {
570 let result = PackStats::validate_header("tests/data/packs/nonexistent.pack");
571 assert!(result.is_err());
572 }
573
574 #[test]
575 fn test_validate_header_invalid_file() {
576 use std::io::Write;
577
578 use tempfile::NamedTempFile;
579
580 let mut temp = NamedTempFile::new().expect("create temp file");
581 temp.write_all(b"XX").expect("write temp file");
582 temp.flush().expect("flush temp file");
583
584 let result = PackStats::validate_header(temp.path());
585 assert!(result.is_err());
586 }
587
588 fn write_pack_object_header(out: &mut Vec<u8>, type_bits: u8, mut size: usize) {
589 let mut byte = ((type_bits & 0x07) << 4) | (size as u8 & 0x0f);
590 size >>= 4;
591 if size != 0 {
592 byte |= 0x80;
593 }
594 out.push(byte);
595
596 while size != 0 {
597 let mut next = (size as u8) & 0x7f;
598 size >>= 7;
599 if size != 0 {
600 next |= 0x80;
601 }
602 out.push(next);
603 }
604 }
605
606 fn append_pack_trailer(bytes: &mut Vec<u8>) {
607 let mut hash = HashAlgorithm::new();
608 hash.update(bytes);
609 let trailer = ObjectHash::from_bytes(&hash.finalize()).expect("pack hash");
610 bytes.extend_from_slice(trailer.as_ref());
611 }
612}