1#![allow(dead_code)]
4
5use std::path::{Path, PathBuf};
6use std::sync::{Arc, Mutex};
7
8use crate::bloom::{
9 bloom_filter_contains, bloom_keyvec_for_path, BloomBuildOutcome, BloomFilterSettings,
10};
11use crate::error::Error;
12use crate::hash::verify_trailer;
13use crate::objects::{HashAlgo, ObjectId};
14use crate::odb::{hash_algo_for_objects_dir, Odb};
15use crate::pack_map::PackData;
16
17fn warn_once_for_disabled_bloom_layer(
18 caches: Option<&crate::repo_caches::RepoCaches>,
19 id: &str,
20) -> bool {
21 let key = format!("bloom-disabled:{id}");
22 caches
23 .map(|c| c.should_warn_commit_graph_once(&key))
24 .unwrap_or(true)
25}
26
27fn warn_once_for_base_chunk_too_small(
28 caches: Option<&crate::repo_caches::RepoCaches>,
29 id: &str,
30) -> bool {
31 let key = format!("base-chunk:{id}");
32 caches
33 .map(|c| c.should_warn_commit_graph_once(&key))
34 .unwrap_or(true)
35}
36
37const SIGNATURE: &[u8; 4] = b"CGPH";
38const GRAPH_VERSION: u8 = 1;
39const COMMIT_GRAPH_MIN_FILE_LEN: u64 = 28;
40
41const CHUNK_OID_FANOUT: u32 = 0x4f49_4446; const CHUNK_OID_LOOKUP: u32 = 0x4f49_444c; const CHUNK_COMMIT_DATA: u32 = 0x4344_4154; const CHUNK_GENERATION_DATA: u32 = 0x4744_4132; const CHUNK_GENERATION_DATA_OVERFLOW: u32 = 0x4744_4f32; const CHUNK_EXTRA_EDGES: u32 = 0x4544_4745; const GRAPH_PARENT_NONE: u32 = 0x7000_0000;
50const GRAPH_EXTRA_EDGES_NEEDED: u32 = 0x8000_0000;
53const GRAPH_LAST_EDGE: u32 = 0x8000_0000;
54const CHUNK_BLOOM_INDEXES: u32 = 0x4249_4458; const CHUNK_BLOOM_DATA: u32 = 0x4244_4154; const CHUNK_BASE_GRAPHS: u32 = 0x4241_5345; const BLOOM_HEADER: usize = crate::bloom::BLOOMDATA_HEADER_LEN;
59
60const CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW: u32 = 1u32 << 31;
62
63fn warn_path_for_graph_file(path: &Path) -> String {
64 let s = path.to_string_lossy();
65 if let Some(idx) = s.find(".git/") {
66 return s[idx..].replace('\\', "/");
67 }
68 s.replace('\\', "/")
69}
70
71#[derive(Debug, Clone)]
73pub struct CommitGraphLayer {
74 pub path: PathBuf,
76 file: Arc<PackData>,
77 body_len: usize,
78 num_commits: u32,
79 oid_lookup_off: usize,
80 chunk_commit_data_off: usize,
81 chunk_generation_data: Option<usize>,
82 chunk_generation_overflow: Option<usize>,
83 read_generation_data: bool,
84 chunk_bloom_indexes: Option<usize>,
85 chunk_bloom_data: Option<(usize, usize)>,
86 bloom_settings: Option<BloomFilterSettings>,
87 bloom_disabled: bool,
88 base_chunk_size: usize,
91 hash_len: usize,
93 algo: HashAlgo,
95 extra_edges_off: Option<usize>,
96 extra_edges_end: Option<usize>,
97}
98
99fn commit_graph_hash_len(hash_version: u8) -> Option<usize> {
101 HashAlgo::try_from(hash_version).ok().map(|a| a.len())
102}
103
104fn commit_graph_chain_hash_is_valid(h: &str, algo: HashAlgo) -> bool {
106 h.len() == algo.hex_len() && h.chars().all(|c| c.is_ascii_hexdigit())
107}
108
109const OIDF_BUCKET_BYTES: usize = 256 * 4;
110
111fn commit_graph_chunk_oob() -> Error {
112 Error::CorruptObject("commit-graph chunk extends past end of file".to_owned())
113}
114
115fn commit_graph_offset_out_of_range() -> Error {
116 Error::CorruptObject("commit-graph chunk offset out of range".to_owned())
117}
118
119fn checked_add_end(start: usize, len: usize, bound: usize) -> Result<usize, Error> {
120 start
121 .checked_add(len)
122 .filter(|&end| end <= bound)
123 .ok_or_else(commit_graph_chunk_oob)
124}
125
126fn body_slice(body: &[u8], start: usize, len: usize) -> Result<&[u8], Error> {
127 let end = checked_add_end(start, len, body.len())?;
128 body.get(start..end).ok_or_else(commit_graph_chunk_oob)
129}
130
131fn parse_toc_offset(body_len: usize, raw: [u8; 8]) -> Result<usize, Error> {
132 let off = u64::from_be_bytes(raw);
133 if off > body_len as u64 {
134 return Err(commit_graph_offset_out_of_range());
135 }
136 usize::try_from(off).map_err(|_| commit_graph_offset_out_of_range())
137}
138
139fn fanout_num_commits(body: &[u8], fanout_off: usize) -> Result<u32, Error> {
140 let fanout = body_slice(body, fanout_off, OIDF_BUCKET_BYTES)?;
141 let word = fanout[255 * 4..256 * 4]
142 .try_into()
143 .map_err(|_| Error::CorruptObject("commit-graph fanout corrupt".to_owned()))?;
144 Ok(u32::from_be_bytes(word))
145}
146
147#[allow(clippy::too_many_arguments)]
148fn validate_single_layer_parent_indices(
149 body: &[u8],
150 num_commits: u32,
151 commit_data_off: usize,
152 graph_data_width: usize,
153 hash_len: usize,
154 extra_edges_off: Option<usize>,
155 toc_entries: &[(u32, usize)],
156 file_end: usize,
157) -> Result<(), Error> {
158 fn chunk_byte_range(
159 start: usize,
160 toc_entries: &[(u32, usize)],
161 file_end: usize,
162 ) -> Result<usize, Error> {
163 let mut ends: Vec<usize> = toc_entries
164 .iter()
165 .map(|&(_, o)| o)
166 .filter(|&o| o > start)
167 .collect();
168 ends.sort_unstable();
169 Ok(ends.first().copied().unwrap_or(file_end))
170 }
171
172 for lex in 0..num_commits {
173 let base = commit_data_off + lex as usize * graph_data_width;
174 let p1_off = base + hash_len;
175 let p1 =
176 u32::from_be_bytes(body[p1_off..p1_off + 4].try_into().map_err(|_| {
177 Error::CorruptObject("commit-graph commit data corrupt".to_owned())
178 })?);
179 let p2 =
180 u32::from_be_bytes(body[p1_off + 4..p1_off + 8].try_into().map_err(|_| {
181 Error::CorruptObject("commit-graph commit data corrupt".to_owned())
182 })?);
183 if p1 != GRAPH_PARENT_NONE
184 && p1 & GRAPH_EXTRA_EDGES_NEEDED == 0
185 && p1 as usize >= num_commits as usize
186 {
187 return Err(Error::CorruptObject(
188 "commit-graph invalid parent index".to_owned(),
189 ));
190 }
191 if p2 != GRAPH_PARENT_NONE {
192 if p2 & GRAPH_EXTRA_EDGES_NEEDED != 0 {
193 let start = (p2 & !GRAPH_EXTRA_EDGES_NEEDED) as usize;
194 let Some(edge_off) = extra_edges_off else {
195 return Err(Error::CorruptObject(
196 "commit-graph extra edge list out of range".to_owned(),
197 ));
198 };
199 let edge_end = chunk_byte_range(edge_off, toc_entries, file_end)?;
200 let edge_len = edge_end.saturating_sub(edge_off);
201 if start * 4 >= edge_len {
202 return Err(Error::CorruptObject(
203 "commit-graph extra edge list out of range".to_owned(),
204 ));
205 }
206 let mut idx = start;
207 loop {
208 if idx * 4 + 4 > edge_len {
209 return Err(Error::CorruptObject(
210 "commit-graph extra edge list corrupt".to_owned(),
211 ));
212 }
213 let ev = u32::from_be_bytes(
214 body[edge_off + idx * 4..edge_off + idx * 4 + 4]
215 .try_into()
216 .map_err(|_| {
217 Error::CorruptObject(
218 "commit-graph extra edge list corrupt".to_owned(),
219 )
220 })?,
221 );
222 let parent = ev & !GRAPH_LAST_EDGE;
223 if parent != GRAPH_PARENT_NONE
224 && parent & GRAPH_EXTRA_EDGES_NEEDED == 0
225 && parent as usize >= num_commits as usize
226 {
227 return Err(Error::CorruptObject(
228 "commit-graph invalid parent index".to_owned(),
229 ));
230 }
231 if ev & GRAPH_LAST_EDGE != 0 {
232 break;
233 }
234 idx += 1;
235 }
236 } else if p2 as usize >= num_commits as usize {
237 return Err(Error::CorruptObject(
238 "commit-graph invalid parent index".to_owned(),
239 ));
240 }
241 }
242 }
243 Ok(())
244}
245
246#[derive(Clone, Copy, PartialEq, Eq)]
247enum CommitGraphParseMode {
248 Open,
250 Verify,
252}
253
254impl CommitGraphLayer {
255 fn body(&self) -> &[u8] {
256 &self.file[..self.body_len]
257 }
258
259 fn try_open(path: PathBuf) -> Result<Self, Error> {
261 let storage = PackData::open_with_min_len(&path, COMMIT_GRAPH_MIN_FILE_LEN)?;
262 Self::try_parse_storage(path, storage, CommitGraphParseMode::Open)
263 }
264
265 pub fn try_parse(path: PathBuf, raw: Vec<u8>) -> Result<Self, Error> {
267 Self::try_parse_storage(
268 path,
269 PackData::from_owned(raw),
270 CommitGraphParseMode::Verify,
271 )
272 }
273
274 fn try_parse_storage(
275 path: PathBuf,
276 storage: Arc<PackData>,
277 mode: CommitGraphParseMode,
278 ) -> Result<Self, Error> {
279 let raw = storage.as_ref();
280 if raw.len() < 28 {
281 return Err(Error::CorruptObject(
282 "commit-graph file too small".to_owned(),
283 ));
284 }
285 let hash_len = match commit_graph_hash_len(raw[5]) {
288 Some(n) => n,
289 None => {
290 return Err(Error::CorruptObject(format!(
291 "commit-graph version/hash not supported (version {} hash {})",
292 raw[4], raw[5]
293 )))
294 }
295 };
296 let algo = HashAlgo::try_from(raw[5]).map_err(|_| {
297 Error::CorruptObject(format!(
298 "commit-graph version/hash not supported (version {} hash {})",
299 raw[4], raw[5]
300 ))
301 })?;
302 if mode == CommitGraphParseMode::Verify {
303 verify_trailer(algo, raw)
304 .map_err(|_| Error::CorruptObject("commit-graph incorrect checksum".to_owned()))?;
305 }
306 let body_len = raw.len().saturating_sub(hash_len);
307 let body = &raw[..body_len];
308 if body.len() < 8 || &body[0..4] != SIGNATURE {
309 return Err(Error::CorruptObject(
310 "commit-graph has bad signature".to_owned(),
311 ));
312 }
313 if body[4] != GRAPH_VERSION {
314 return Err(Error::CorruptObject(format!(
315 "commit-graph version/hash not supported (version {} hash {})",
316 body[4], body[5]
317 )));
318 }
319 let num_chunks = body[6] as usize;
320 let toc_start: usize = 8;
321 let toc_end = num_chunks
322 .checked_add(1)
323 .and_then(|n| n.checked_mul(12))
324 .and_then(|n| toc_start.checked_add(n))
325 .ok_or_else(|| {
326 Error::CorruptObject("commit-graph truncated at chunk table".to_owned())
327 })?;
328 if body.len() < toc_end {
329 return Err(Error::CorruptObject(
330 "commit-graph truncated at chunk table".to_owned(),
331 ));
332 }
333
334 let mut fanout_off = None;
335 let mut oid_lookup_off = None;
336 let mut commit_data_off = None;
337 let mut generation_off = None;
338 let mut generation_overflow_off = None;
339 let mut bloom_idx_off = None;
340 let mut bloom_data_range = None;
341 let mut base_graphs_off = None;
342 let mut extra_edges_off = None;
343 let mut extra_edges_end = None;
344 let mut chunk_offsets: Vec<usize> = Vec::new();
345 let mut toc_entries: Vec<(u32, usize)> = Vec::with_capacity(num_chunks);
346
347 for i in 0..num_chunks {
348 let e = toc_start + i * 12;
349 let id = u32::from_be_bytes(
350 body[e..e + 4]
351 .try_into()
352 .map_err(|_| Error::CorruptObject("commit-graph bad TOC".to_owned()))?,
353 );
354 let off = parse_toc_offset(
355 body.len(),
356 body[e + 4..e + 12]
357 .try_into()
358 .map_err(|_| Error::CorruptObject("commit-graph bad TOC".to_owned()))?,
359 )?;
360 toc_entries.push((id, off));
361 chunk_offsets.push(off);
362 match id {
363 CHUNK_OID_FANOUT => fanout_off = Some(off),
364 CHUNK_OID_LOOKUP => oid_lookup_off = Some(off),
365 CHUNK_COMMIT_DATA => commit_data_off = Some(off),
366 CHUNK_GENERATION_DATA => generation_off = Some(off),
367 CHUNK_GENERATION_DATA_OVERFLOW => generation_overflow_off = Some(off),
368 CHUNK_BLOOM_INDEXES => bloom_idx_off = Some(off),
369 CHUNK_EXTRA_EDGES => {
370 extra_edges_off = Some(off);
371 extra_edges_end = Some(if i + 1 < num_chunks {
372 let e2 = toc_start
373 .checked_add((i + 1).checked_mul(12).ok_or_else(|| {
374 Error::CorruptObject("commit-graph bad TOC".to_owned())
375 })?)
376 .ok_or_else(|| {
377 Error::CorruptObject("commit-graph bad TOC".to_owned())
378 })?;
379 parse_toc_offset(
380 body.len(),
381 body[e2 + 4..e2 + 12].try_into().map_err(|_| {
382 Error::CorruptObject("commit-graph bad TOC".to_owned())
383 })?,
384 )?
385 } else {
386 let term = toc_start
387 .checked_add(num_chunks.checked_mul(12).ok_or_else(|| {
388 Error::CorruptObject("commit-graph bad TOC".to_owned())
389 })?)
390 .ok_or_else(|| {
391 Error::CorruptObject("commit-graph bad TOC".to_owned())
392 })?;
393 parse_toc_offset(
394 body.len(),
395 body[term + 4..term + 12].try_into().map_err(|_| {
396 Error::CorruptObject("commit-graph bad TOC".to_owned())
397 })?,
398 )?
399 });
400 }
401 CHUNK_BASE_GRAPHS => base_graphs_off = Some(off),
402 CHUNK_BLOOM_DATA => {
403 let end = if i + 1 < num_chunks {
404 let e2 = toc_start
405 .checked_add((i + 1).checked_mul(12).ok_or_else(|| {
406 Error::CorruptObject("commit-graph bad TOC".to_owned())
407 })?)
408 .ok_or_else(|| {
409 Error::CorruptObject("commit-graph bad TOC".to_owned())
410 })?;
411 parse_toc_offset(
412 body.len(),
413 body[e2 + 4..e2 + 12].try_into().map_err(|_| {
414 Error::CorruptObject("commit-graph bad TOC".to_owned())
415 })?,
416 )?
417 } else {
418 let term = toc_start
419 .checked_add(num_chunks.checked_mul(12).ok_or_else(|| {
420 Error::CorruptObject("commit-graph bad TOC".to_owned())
421 })?)
422 .ok_or_else(|| {
423 Error::CorruptObject("commit-graph bad TOC".to_owned())
424 })?;
425 parse_toc_offset(
426 body.len(),
427 body[term + 4..term + 12].try_into().map_err(|_| {
428 Error::CorruptObject("commit-graph bad TOC".to_owned())
429 })?,
430 )?
431 };
432 let len = end.checked_sub(off).ok_or_else(|| {
433 Error::CorruptObject("commit-graph chunk layout invalid".to_owned())
434 })?;
435 bloom_data_range = Some((off, len));
436 }
437 _ => {}
438 }
439 }
440 let file_end_pos = toc_start
441 .checked_add(
442 num_chunks
443 .checked_mul(12)
444 .ok_or_else(|| Error::CorruptObject("commit-graph bad file end".to_owned()))?,
445 )
446 .and_then(|t| t.checked_add(4))
447 .ok_or_else(|| Error::CorruptObject("commit-graph bad file end".to_owned()))?;
448 let file_end = parse_toc_offset(
449 body.len(),
450 body[file_end_pos..file_end_pos + 8]
451 .try_into()
452 .map_err(|_| Error::CorruptObject("commit-graph bad file end".to_owned()))?,
453 )?;
454 if file_end > body.len() {
455 return Err(commit_graph_offset_out_of_range());
456 }
457 chunk_offsets.push(file_end);
458 chunk_offsets.sort_unstable();
459 chunk_offsets.dedup();
460
461 fn chunk_byte_range(
462 start: usize,
463 toc_entries: &[(u32, usize)],
464 file_end: usize,
465 ) -> Result<usize, Error> {
466 let mut ends: Vec<usize> = toc_entries
467 .iter()
468 .map(|&(_, o)| o)
469 .filter(|&o| o > start)
470 .collect();
471 ends.sort_unstable();
472 let end = ends.first().copied().unwrap_or(file_end);
473 if end < start {
474 return Err(Error::CorruptObject(
475 "commit-graph chunk layout invalid".to_owned(),
476 ));
477 }
478 Ok(end)
479 }
480
481 if mode == CommitGraphParseMode::Verify {
482 if let Some(gda) = generation_off {
483 let gda_end = chunk_byte_range(gda, &toc_entries, file_end)?;
484 let gda_len = gda_end.saturating_sub(gda);
485 let fo = fanout_off.ok_or_else(|| {
486 Error::CorruptObject("commit-graph missing fanout".to_owned())
487 })?;
488 let num_commits_probe = fanout_num_commits(body, fo)?;
489 let expected = num_commits_probe as usize * 4;
490 if gda_len < expected {
491 return Err(Error::CorruptObject(
492 "commit-graph generation data chunk is too small".to_owned(),
493 ));
494 }
495 let gda_slice = body.get(gda..gda + expected).ok_or_else(|| {
496 Error::CorruptObject("commit-graph generation data OOB".to_owned())
497 })?;
498 let mut max_overflow_idx: Option<u32> = None;
499 for w in 0..num_commits_probe as usize {
500 let v = u32::from_be_bytes(gda_slice[w * 4..w * 4 + 4].try_into().map_err(
501 |_| Error::CorruptObject("commit-graph GDA2 corrupt".to_owned()),
502 )?);
503 if v & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW != 0 {
504 let pos = v ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW;
505 max_overflow_idx = Some(match max_overflow_idx {
506 None => pos,
507 Some(m) => m.max(pos),
508 });
509 }
510 }
511 if let Some(pos) = max_overflow_idx {
512 let Some(gdo_start) = generation_overflow_off else {
513 return Err(Error::CorruptObject(
514 "commit-graph requires overflow generation data but has none"
515 .to_owned(),
516 ));
517 };
518 let gdo_end = chunk_byte_range(gdo_start, &toc_entries, file_end)?;
519 let overflow_bytes = gdo_end.saturating_sub(gdo_start);
520 let n_slots = overflow_bytes / 8;
521 if n_slots <= pos as usize {
522 return Err(Error::CorruptObject(
523 "commit-graph overflow generation data is too small".to_owned(),
524 ));
525 }
526 }
527 }
528 }
529 let bidx_len = bloom_idx_off.and_then(|b| {
530 chunk_offsets
531 .iter()
532 .find(|&&o| o > b)
533 .map(|&next| next.saturating_sub(b))
534 });
535
536 let fanout_off = fanout_off.ok_or_else(|| {
537 Error::CorruptObject("commit-graph missing OID fanout chunk".to_owned())
538 })?;
539 let oid_lookup_off = oid_lookup_off.ok_or_else(|| {
540 Error::CorruptObject("commit-graph missing OID lookup chunk".to_owned())
541 })?;
542 let commit_data_off = commit_data_off.ok_or_else(|| {
543 Error::CorruptObject("commit-graph missing commit data chunk".to_owned())
544 })?;
545 let num_commits = fanout_num_commits(body, fanout_off)?;
546 let oid_lookup_bytes = (num_commits as usize)
547 .checked_mul(hash_len)
548 .ok_or_else(commit_graph_chunk_oob)?;
549 checked_add_end(oid_lookup_off, oid_lookup_bytes, body.len())?;
550 let graph_data_width = hash_len
551 .checked_add(16)
552 .ok_or_else(commit_graph_chunk_oob)?;
553 let cdat_bytes = (num_commits as usize)
554 .checked_mul(graph_data_width)
555 .ok_or_else(commit_graph_chunk_oob)?;
556 checked_add_end(commit_data_off, cdat_bytes, body.len())?;
557
558 if mode == CommitGraphParseMode::Verify {
559 for bucket in 0usize..255 {
560 let a_off = fanout_off
561 .checked_add(bucket.checked_mul(4).ok_or_else(commit_graph_chunk_oob)?)
562 .ok_or_else(commit_graph_chunk_oob)?;
563 let a =
564 u32::from_be_bytes(body_slice(body, a_off, 4)?.try_into().map_err(|_| {
565 Error::CorruptObject("commit-graph fanout corrupt".to_owned())
566 })?);
567 let b_off = fanout_off
568 .checked_add(
569 (bucket + 1)
570 .checked_mul(4)
571 .ok_or_else(commit_graph_chunk_oob)?,
572 )
573 .ok_or_else(commit_graph_chunk_oob)?;
574 let b =
575 u32::from_be_bytes(body_slice(body, b_off, 4)?.try_into().map_err(|_| {
576 Error::CorruptObject("commit-graph fanout corrupt".to_owned())
577 })?);
578 if a > b {
579 return Err(Error::CorruptObject(
580 "commit-graph fanout value non-monotonic".to_owned(),
581 ));
582 }
583 }
584
585 for lex in 1..num_commits {
586 let off_a = oid_lookup_off
587 .checked_add(
588 (lex as usize - 1)
589 .checked_mul(hash_len)
590 .ok_or_else(commit_graph_chunk_oob)?,
591 )
592 .ok_or_else(commit_graph_chunk_oob)?;
593 let off_b = oid_lookup_off
594 .checked_add(
595 (lex as usize)
596 .checked_mul(hash_len)
597 .ok_or_else(commit_graph_chunk_oob)?,
598 )
599 .ok_or_else(commit_graph_chunk_oob)?;
600 let slice_a = body_slice(body, off_a, hash_len)?;
601 let slice_b = body_slice(body, off_b, hash_len)?;
602 if slice_a >= slice_b {
603 return Err(Error::CorruptObject(
604 "commit-graph incorrect OID order".to_owned(),
605 ));
606 }
607 }
608
609 if base_graphs_off.is_none() {
610 validate_single_layer_parent_indices(
611 body,
612 num_commits,
613 commit_data_off,
614 graph_data_width,
615 hash_len,
616 extra_edges_off,
617 &toc_entries,
618 file_end,
619 )?;
620 }
621 }
622
623 let read_generation_data = generation_off.is_some();
624 let mut bloom_settings = None;
625 let mut chunk_bloom_data = None;
626 if let (Some(_bidx), Some((bdat_off, bdat_len))) = (bloom_idx_off, bloom_data_range) {
627 if bdat_len < BLOOM_HEADER {
628 } else if bdat_off + bdat_len <= body.len() {
629 let hdr = &body[bdat_off..bdat_off + BLOOM_HEADER];
630 let hash_version: [u8; 4] = hdr[0..4]
631 .try_into()
632 .map_err(|_| Error::CorruptObject("Bloom header corrupt".to_owned()))?;
633 let num_hashes: [u8; 4] = hdr[4..8]
634 .try_into()
635 .map_err(|_| Error::CorruptObject("Bloom header corrupt".to_owned()))?;
636 let bits_per_entry: [u8; 4] = hdr[8..12]
637 .try_into()
638 .map_err(|_| Error::CorruptObject("Bloom header corrupt".to_owned()))?;
639 bloom_settings = Some(BloomFilterSettings {
640 hash_version: u32::from_be_bytes(hash_version),
641 num_hashes: u32::from_be_bytes(num_hashes),
642 bits_per_entry: u32::from_be_bytes(bits_per_entry),
643 max_changed_paths: 512,
644 });
645 chunk_bloom_data = Some((bdat_off, bdat_len));
646 }
647 }
648
649 let bloom_indexes_ok = if let (Some(bidx), Some(bsize)) = (bloom_idx_off, bidx_len) {
650 !(bsize / 4 != num_commits as usize || bidx + bsize > body.len())
651 } else {
652 false
653 };
654 let bloom_pair_ok = bloom_settings.is_some()
655 && chunk_bloom_data.is_some()
656 && bloom_indexes_ok
657 && chunk_bloom_data.is_some_and(|(_, len)| len >= BLOOM_HEADER);
658
659 let (chunk_bloom_indexes, bloom_settings) = if bloom_pair_ok {
660 (bloom_idx_off, bloom_settings)
661 } else {
662 (None, None)
663 };
664 let chunk_bloom_data = if bloom_pair_ok {
665 chunk_bloom_data
666 } else {
667 None
668 };
669
670 let base_chunk_size = match base_graphs_off {
671 Some(off) => {
672 let end = chunk_byte_range(off, &toc_entries, file_end)?;
673 end.saturating_sub(off)
674 }
675 None => 0,
676 };
677
678 Ok(Self {
679 path,
680 file: storage,
681 body_len,
682 num_commits,
683 oid_lookup_off,
684 chunk_commit_data_off: commit_data_off,
685 chunk_generation_data: generation_off,
686 chunk_generation_overflow: generation_overflow_off,
687 read_generation_data,
688 chunk_bloom_indexes,
689 chunk_bloom_data,
690 bloom_settings,
691 bloom_disabled: false,
692 base_chunk_size,
693 hash_len,
694 algo,
695 extra_edges_off,
696 extra_edges_end,
697 })
698 }
699
700 pub(crate) fn parse(path: PathBuf, raw: Vec<u8>) -> Option<Self> {
701 Self::try_parse(path, raw).ok()
702 }
703
704 fn oid_at_lex(&self, lex_index: u32) -> Option<ObjectId> {
705 if lex_index >= self.num_commits {
706 return None;
707 }
708 let off = self.oid_lookup_off + lex_index as usize * self.hash_len;
709 ObjectId::from_bytes(self.body().get(off..off + self.hash_len)?).ok()
710 }
711
712 fn parent_global_positions(&self, lex_index: u32) -> Option<Vec<u32>> {
719 let (p1, p2, _) = self.cdat_record(lex_index)?;
720 let mut out = Vec::new();
721 if p1 == GRAPH_PARENT_NONE {
722 return Some(out);
723 }
724 if p1 & GRAPH_EXTRA_EDGES_NEEDED != 0 {
725 return None;
726 }
727 out.push(p1);
728 if p2 == GRAPH_PARENT_NONE {
729 return Some(out);
730 }
731 if p2 & GRAPH_EXTRA_EDGES_NEEDED != 0 {
732 let start = (p2 & !GRAPH_EXTRA_EDGES_NEEDED) as usize;
733 let edge_off = self.extra_edges_off?;
734 let edge_end = self.extra_edges_end?;
735 let edge_len = edge_end.saturating_sub(edge_off);
736 if start.saturating_mul(4) >= edge_len {
737 return None;
738 }
739 let mut idx = start;
740 loop {
741 if idx.saturating_mul(4).saturating_add(4) > edge_len {
742 return None;
743 }
744 let ev = u32::from_be_bytes(
745 self.body()[edge_off + idx * 4..edge_off + idx * 4 + 4]
746 .try_into()
747 .ok()?,
748 );
749 let parent = ev & !GRAPH_LAST_EDGE;
750 if parent != GRAPH_PARENT_NONE && parent & GRAPH_EXTRA_EDGES_NEEDED == 0 {
751 out.push(parent);
752 }
753 if ev & GRAPH_LAST_EDGE != 0 {
754 break;
755 }
756 idx += 1;
757 }
758 return Some(out);
759 }
760 out.push(p2);
761 Some(out)
762 }
763
764 fn cdat_root_tree(&self, lex_index: u32) -> Option<ObjectId> {
766 if lex_index >= self.num_commits {
767 return None;
768 }
769 let base = self.chunk_commit_data_off + lex_index as usize * (self.hash_len + 16);
770 ObjectId::from_bytes(self.body().get(base..base + self.hash_len)?).ok()
771 }
772
773 fn cdat_record(&self, lex_index: u32) -> Option<(u32, u32, i64)> {
774 if lex_index >= self.num_commits {
775 return None;
776 }
777 let base = self.chunk_commit_data_off + lex_index as usize * (self.hash_len + 16);
778 let p1_off = base + self.hash_len;
779 let p1 = u32::from_be_bytes(self.body().get(p1_off..p1_off + 4)?.try_into().ok()?);
780 let p2 = u32::from_be_bytes(self.body().get(p1_off + 4..p1_off + 8)?.try_into().ok()?);
781 let gen_time =
782 u64::from_be_bytes(self.body().get(p1_off + 8..p1_off + 16)?.try_into().ok()?);
783 let commit_time = (gen_time & 0x3_FFFF_FFFF) as i64;
784 Some((p1, p2, commit_time))
785 }
786
787 fn corrected_commit_generation(&self, lex_index: u32) -> Option<u64> {
789 if !self.read_generation_data {
790 return None;
791 }
792 let (_, _, commit_time) = self.cdat_record(lex_index)?;
793 let gda_off = self.chunk_generation_data?;
794 if lex_index >= self.num_commits {
795 return None;
796 }
797 let start = gda_off + lex_index as usize * 4;
798 let word = u32::from_be_bytes(self.body().get(start..start + 4)?.try_into().ok()?);
799 let offset = if word & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW != 0 {
800 let slot = (word ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) as usize;
801 let gdo_off = self.chunk_generation_overflow?;
802 let off = gdo_off + slot * 8;
803 let hi = u32::from_be_bytes(self.body().get(off..off + 4)?.try_into().ok()?);
804 let lo = u32::from_be_bytes(self.body().get(off + 4..off + 8)?.try_into().ok()?);
805 (u64::from(hi) << 32) | u64::from(lo)
806 } else {
807 u64::from(word)
808 };
809 Some((commit_time as u64).saturating_add(offset))
810 }
811
812 fn bsearch_oid(&self, oid: &ObjectId) -> Option<u32> {
813 let mut lo = 0u32;
814 let mut hi = self.num_commits;
815 let bytes = oid.as_bytes();
816 while lo < hi {
817 let mid = (lo + hi) / 2;
818 let off = self.oid_lookup_off + mid as usize * self.hash_len;
819 let slice = &self.body()[off..off + self.hash_len];
820 match slice.cmp(bytes) {
821 std::cmp::Ordering::Less => lo = mid + 1,
822 std::cmp::Ordering::Greater => hi = mid,
823 std::cmp::Ordering::Equal => return Some(mid),
824 }
825 }
826 None
827 }
828
829 fn disable_bloom(&mut self) {
830 self.chunk_bloom_indexes = None;
831 self.chunk_bloom_data = None;
832 self.bloom_settings = None;
833 self.bloom_disabled = true;
834 }
835
836 fn layer_display_id(&self) -> String {
837 self.path
838 .file_stem()
839 .and_then(|s| s.to_str())
840 .map(|s| s.strip_prefix("graph-").unwrap_or(s).to_string())
841 .unwrap_or_else(|| "commit-graph".to_string())
842 }
843
844 fn bloom_filter_slice(&self, lex_index: u32) -> Option<&[u8]> {
845 let _settings = self.bloom_settings.as_ref()?;
846 let bidx_base = self.chunk_bloom_indexes?;
847 let (bdat_off, bdat_total) = self.chunk_bloom_data?;
848 let _graph_warn = warn_path_for_graph_file(self.path.as_path());
849 if lex_index >= self.num_commits {
850 return None;
851 }
852 let payload_len = bdat_total.saturating_sub(BLOOM_HEADER);
853 let end_rel = u32::from_be_bytes(
854 self.body()[bidx_base + lex_index as usize * 4..bidx_base + lex_index as usize * 4 + 4]
855 .try_into()
856 .ok()?,
857 ) as usize;
858 let start_rel = if lex_index == 0 {
859 0usize
860 } else {
861 u32::from_be_bytes(
862 self.body()[bidx_base + (lex_index as usize - 1) * 4
863 ..bidx_base + (lex_index as usize - 1) * 4 + 4]
864 .try_into()
865 .ok()?,
866 ) as usize
867 };
868 let max_payload = payload_len;
874 if end_rel > max_payload {
875 return None;
876 }
877 if start_rel > max_payload {
878 return None;
879 }
880 if end_rel < start_rel {
881 return None;
882 }
883 let data_base = bdat_off + BLOOM_HEADER;
884 let abs_start = data_base + start_rel;
885 let abs_end = data_base + end_rel;
886 if abs_end > bdat_off + bdat_total || abs_start > abs_end {
887 return None;
888 }
889 Some(&self.body()[abs_start..abs_end])
890 }
891}
892
893#[derive(Debug, Clone, Copy, PartialEq, Eq)]
895pub enum BloomPrecheck {
896 Inapplicable,
898 NotInGraph,
900 FilterNotPresent,
902 DefinitelyNot,
904 Maybe,
906}
907
908#[derive(Debug, Default, Clone)]
910pub struct BloomWalkStats {
911 pub filter_not_present: u32,
913 pub maybe: u32,
915 pub definitely_not: u32,
917 pub false_positive: u32,
919}
920
921impl BloomWalkStats {
922 pub fn record_precheck(&mut self, pre: BloomPrecheck) {
924 match pre {
925 BloomPrecheck::Inapplicable | BloomPrecheck::NotInGraph => {}
926 BloomPrecheck::FilterNotPresent => self.filter_not_present += 1,
927 BloomPrecheck::DefinitelyNot => self.definitely_not += 1,
928 BloomPrecheck::Maybe => self.maybe += 1,
929 }
930 }
931
932 pub fn record_false_positive(&mut self) {
934 self.false_positive += 1;
935 }
936}
937
938pub type BloomWalkStatsHandle = Arc<Mutex<BloomWalkStats>>;
940
941pub(crate) const UNRANKED_GENERATION: u64 = (1u64 << 63) - 1;
944
945#[derive(Debug, Clone)]
947pub(crate) struct GraphWalkEntry {
948 pub(crate) tree: ObjectId,
950 pub(crate) parents: Vec<ObjectId>,
952 pub(crate) time: i64,
954 pub(crate) generation: u64,
956}
957
958#[derive(Debug, Clone)]
960pub struct CommitGraphChain {
961 layers: Vec<CommitGraphLayer>,
962}
963
964#[cfg(any(test, feature = "test-support"))]
966pub(crate) fn commit_graph_chain_from_layers(layers: Vec<CommitGraphLayer>) -> CommitGraphChain {
967 CommitGraphChain { layers }
968}
969
970impl CommitGraphChain {
971 pub fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> {
989 if bytes.get(7).is_some_and(|&base_graphs| base_graphs != 0) {
990 return Err(Error::CorruptObject(
991 "commit-graph layer depends on base graphs".to_owned(),
992 ));
993 }
994 let layer = CommitGraphLayer::try_parse(PathBuf::from("commit-graph"), bytes)?;
995 if layer.base_chunk_size != 0 {
996 return Err(Error::CorruptObject(
997 "commit-graph layer depends on base graphs".to_owned(),
998 ));
999 }
1000 Ok(Self {
1001 layers: vec![layer],
1002 })
1003 }
1004
1005 #[must_use]
1007 pub fn hash_algo(&self) -> HashAlgo {
1008 self.layers
1009 .first()
1010 .map_or(HashAlgo::Sha1, |layer| layer.algo)
1011 }
1012
1013 #[must_use]
1015 pub fn top_layer_bloom_settings(&self) -> Option<BloomFilterSettings> {
1016 self.layers.first()?.bloom_settings
1017 }
1018
1019 #[must_use]
1021 pub fn total_commits(&self) -> u32 {
1022 self.layers.iter().map(|l| l.num_commits).sum()
1023 }
1024
1025 #[must_use]
1027 pub fn layer_paths_oldest_first(&self) -> Vec<PathBuf> {
1028 self.layers.iter().rev().map(|l| l.path.clone()).collect()
1029 }
1030
1031 #[must_use]
1033 pub fn num_layers(&self) -> usize {
1034 self.layers.len()
1035 }
1036
1037 #[must_use]
1039 pub fn layer_commit_counts_tip_first(&self) -> Vec<u32> {
1040 self.layers.iter().map(|l| l.num_commits).collect()
1041 }
1042
1043 #[must_use]
1045 pub fn layer_has_generation_data_tip_first(&self) -> Vec<bool> {
1046 self.layers
1047 .iter()
1048 .map(|l| l.chunk_generation_data.is_some())
1049 .collect()
1050 }
1051
1052 #[must_use]
1054 pub fn layer_hashes_tip_first(&self) -> Vec<String> {
1055 self.layers.iter().map(|l| l.layer_display_id()).collect()
1056 }
1057
1058 #[must_use]
1062 pub fn layer_object_dirs_tip_first(&self) -> Vec<PathBuf> {
1063 self.layers
1064 .iter()
1065 .map(|l| {
1066 let p = l.path.as_path();
1069 let info = if p
1070 .parent()
1071 .and_then(|d| d.file_name())
1072 .map(|n| n == "commit-graphs")
1073 .unwrap_or(false)
1074 {
1075 p.parent().and_then(|d| d.parent())
1076 } else {
1077 p.parent()
1078 };
1079 info.and_then(|d| d.parent())
1080 .map(Path::to_path_buf)
1081 .unwrap_or_else(|| PathBuf::from("."))
1082 })
1083 .collect()
1084 }
1085
1086 #[must_use]
1090 pub fn sub_chain_tip_first(&self, start: usize, end: usize) -> Option<Self> {
1091 let end = end.min(self.layers.len());
1092 if start >= end {
1093 return None;
1094 }
1095 Some(Self {
1096 layers: self.layers[start..end].to_vec(),
1097 })
1098 }
1099
1100 #[must_use]
1102 pub fn layer_oids(&self, tip_first_idx: usize) -> Vec<ObjectId> {
1103 let Some(layer) = self.layers.get(tip_first_idx) else {
1104 return Vec::new();
1105 };
1106 let mut out = Vec::with_capacity(layer.num_commits as usize);
1107 for i in 0..layer.num_commits {
1108 if let Some(oid) = layer.oid_at_lex(i) {
1109 out.push(oid);
1110 }
1111 }
1112 out
1113 }
1114
1115 pub fn try_load(objects_dir: &Path) -> Result<Option<Self>, Error> {
1120 Self::try_load_with_caches(objects_dir, None)
1121 }
1122
1123 pub fn try_load_with_caches(
1125 objects_dir: &Path,
1126 caches: Option<&crate::repo_caches::RepoCaches>,
1127 ) -> Result<Option<Self>, Error> {
1128 let info = objects_dir.join("info");
1129 let chain_path = info.join("commit-graphs").join("commit-graph-chain");
1130 if chain_path.is_file() {
1131 let algo =
1132 hash_algo_for_objects_dir(&crate::environment::Environment::empty(), objects_dir);
1133 let content = std::fs::read_to_string(&chain_path).map_err(Error::from)?;
1134 let mut layers = Vec::new();
1135 for line in content.lines() {
1136 let h = line.trim();
1137 if !commit_graph_chain_hash_is_valid(h, algo) {
1138 continue;
1139 }
1140 let graph_path = info.join("commit-graphs").join(format!("graph-{h}.graph"));
1141 let layer = CommitGraphLayer::try_open(graph_path)?;
1142 let n = layers.len();
1149 if n > 0 && layer.base_chunk_size / layer.hash_len < n {
1150 warn_once_for_base_chunk_too_small(caches, &layer.layer_display_id());
1151 break;
1152 }
1153 layers.push(layer);
1154 }
1155 if layers.is_empty() {
1156 return Ok(None);
1157 }
1158 layers.reverse();
1162 let mut chain = Self { layers };
1163 chain.validate_bloom_compatibility(caches);
1164 return Ok(Some(chain));
1165 }
1166 let single = info.join("commit-graph");
1167 if single.is_file() {
1168 let layer = CommitGraphLayer::try_open(single.clone())?;
1169 let mut chain = Self {
1170 layers: vec![layer],
1171 };
1172 chain.validate_bloom_compatibility(caches);
1173 return Ok(Some(chain));
1174 }
1175 Ok(None)
1176 }
1177
1178 pub fn load(objects_dir: &Path) -> Option<Self> {
1180 Self::try_load(objects_dir).ok().flatten()
1181 }
1182
1183 pub fn verify_on_disk(objects_dir: &Path) -> Result<(), Error> {
1191 let Some(chain) = Self::try_load(objects_dir)? else {
1192 return Ok(());
1193 };
1194 for path in chain.layer_paths_oldest_first() {
1195 let raw = std::fs::read(&path).map_err(Error::from)?;
1196 CommitGraphLayer::try_parse(path, raw)?;
1197 }
1198 Ok(())
1199 }
1200
1201 pub fn try_load_across(
1225 objects_dir: &Path,
1226 alt_dirs: &[PathBuf],
1227 ) -> Result<Option<Self>, Error> {
1228 Self::try_load_across_with_caches(objects_dir, alt_dirs, None)
1229 }
1230
1231 pub fn try_load_across_with_caches(
1237 objects_dir: &Path,
1238 alt_dirs: &[PathBuf],
1239 caches: Option<&crate::repo_caches::RepoCaches>,
1240 ) -> Result<Option<Self>, Error> {
1241 let layer_dir = |dir: &Path| dir.join("info").join("commit-graphs");
1242 let resolve_layer = |hash: &str| -> Option<PathBuf> {
1243 let name = format!("graph-{hash}.graph");
1244 let local = layer_dir(objects_dir).join(&name);
1245 if local.is_file() {
1246 return Some(local);
1247 }
1248 alt_dirs
1249 .iter()
1250 .map(|d| layer_dir(d).join(&name))
1251 .find(|p| p.is_file())
1252 };
1253
1254 let local_chain = layer_dir(objects_dir).join("commit-graph-chain");
1262 let local_single = objects_dir.join("info").join("commit-graph");
1263 let chain_path = if local_chain.is_file() {
1264 local_chain
1265 } else if local_single.is_file() {
1266 return Self::try_load_with_caches(objects_dir, caches);
1267 } else {
1268 match alt_dirs
1269 .iter()
1270 .map(PathBuf::as_path)
1271 .find(|d| layer_dir(d).join("commit-graph-chain").is_file())
1272 {
1273 Some(owner) => layer_dir(owner).join("commit-graph-chain"),
1274 None => return Ok(None),
1275 }
1276 };
1277
1278 let algo =
1279 hash_algo_for_objects_dir(&crate::environment::Environment::empty(), objects_dir);
1280 let content = std::fs::read_to_string(&chain_path).map_err(Error::from)?;
1281 let mut layers = Vec::new();
1282 for line in content.lines() {
1283 let h = line.trim();
1284 if !commit_graph_chain_hash_is_valid(h, algo) {
1285 continue;
1286 }
1287 let graph_path = resolve_layer(h).ok_or_else(|| {
1288 Error::from(std::io::Error::new(
1289 std::io::ErrorKind::NotFound,
1290 format!("commit-graph layer graph-{h}.graph not found"),
1291 ))
1292 })?;
1293 let layer = CommitGraphLayer::try_open(graph_path)?;
1294 let n = layers.len();
1295 if n > 0 && layer.base_chunk_size / layer.hash_len < n {
1296 warn_once_for_base_chunk_too_small(caches, &layer.layer_display_id());
1297 break;
1298 }
1299 layers.push(layer);
1300 }
1301 if layers.is_empty() {
1302 return Ok(None);
1303 }
1304 layers.reverse();
1305 let mut chain = Self { layers };
1306 chain.validate_bloom_compatibility(caches);
1307 Ok(Some(chain))
1308 }
1309
1310 fn validate_bloom_compatibility(&mut self, caches: Option<&crate::repo_caches::RepoCaches>) {
1311 let mut ref_settings: Option<BloomFilterSettings> = None;
1317 for layer in &mut self.layers {
1318 let Some(bs) = layer.bloom_settings else {
1319 continue;
1320 };
1321 match ref_settings {
1322 None => ref_settings = Some(bs),
1323 Some(r) => {
1324 if r.hash_version != bs.hash_version
1325 || r.num_hashes != bs.num_hashes
1326 || r.bits_per_entry != bs.bits_per_entry
1327 {
1328 let id = layer.layer_display_id();
1329 warn_once_for_disabled_bloom_layer(caches, &id);
1335 layer.disable_bloom();
1336 }
1337 }
1338 }
1339 }
1340 }
1341
1342 pub fn existing_filter_bytes(
1348 &self,
1349 oid: &ObjectId,
1350 want: &BloomFilterSettings,
1351 ) -> Option<Vec<u8>> {
1352 let (layer_idx, lex) = self.find_commit(oid)?;
1353 let layer = &self.layers[layer_idx];
1354 let settings = layer.bloom_settings.as_ref()?;
1355 if settings.hash_version != want.hash_version
1356 || settings.num_hashes != want.num_hashes
1357 || settings.bits_per_entry != want.bits_per_entry
1358 {
1359 return None;
1360 }
1361 match layer.bloom_filter_slice(lex) {
1368 Some(s) if !s.is_empty() => Some(s.to_vec()),
1369 _ => None,
1370 }
1371 }
1372
1373 pub fn upgradable_filter_bytes(
1379 &self,
1380 oid: &ObjectId,
1381 want: &BloomFilterSettings,
1382 ) -> Option<Vec<u8>> {
1383 let (layer_idx, lex) = self.find_commit(oid)?;
1384 let layer = &self.layers[layer_idx];
1385 let settings = layer.bloom_settings.as_ref()?;
1386 if settings.hash_version == want.hash_version {
1387 return None;
1388 }
1389 if settings.num_hashes != want.num_hashes || settings.bits_per_entry != want.bits_per_entry
1390 {
1391 return None;
1392 }
1393 match layer.bloom_filter_slice(lex) {
1394 Some(s) if !s.is_empty() => Some(s.to_vec()),
1395 _ => None,
1396 }
1397 }
1398
1399 pub(crate) fn bloom_settings_for_commit(&self, oid: &ObjectId) -> Option<BloomFilterSettings> {
1401 let (layer_idx, _) = self.find_commit(oid)?;
1402 self.layers[layer_idx].bloom_settings
1403 }
1404
1405 pub fn find_commit(&self, oid: &ObjectId) -> Option<(usize, u32)> {
1407 for (i, layer) in self.layers.iter().enumerate() {
1408 if let Some(lex) = layer.bsearch_oid(oid) {
1409 return Some((i, lex));
1410 }
1411 }
1412 None
1413 }
1414
1415 pub fn global_position(&self, oid: &ObjectId) -> Option<u32> {
1417 let (layer_idx, lex) = self.find_commit(oid)?;
1418 let below: u32 = self.layers[layer_idx + 1..]
1419 .iter()
1420 .map(|l| l.num_commits)
1421 .sum();
1422 Some(below + lex)
1423 }
1424
1425 fn oid_at_global_position(&self, pos: u32) -> Option<ObjectId> {
1428 let mut remaining = pos;
1430 for layer in self.layers.iter().rev() {
1431 if remaining < layer.num_commits {
1432 return layer.oid_at_lex(remaining);
1433 }
1434 remaining -= layer.num_commits;
1435 }
1436 None
1437 }
1438
1439 pub fn graph_commit(&self, oid: &ObjectId) -> Option<(Vec<ObjectId>, i64)> {
1442 let (layer_idx, lex) = self.find_commit(oid)?;
1443 self.graph_commit_at(layer_idx, lex)
1444 }
1445
1446 fn graph_commit_at(&self, layer_idx: usize, lex: u32) -> Option<(Vec<ObjectId>, i64)> {
1447 let layer = &self.layers[layer_idx];
1448 let (_, _, commit_time) = layer.cdat_record(lex)?;
1449 let oids = layer
1450 .parent_global_positions(lex)?
1451 .into_iter()
1452 .filter_map(|pos| self.oid_at_global_position(pos))
1453 .collect();
1454 Some((oids, commit_time))
1455 }
1456
1457 pub(crate) fn graph_commit_tree(&self, oid: &ObjectId) -> Option<ObjectId> {
1459 let (layer_idx, lex) = self.find_commit(oid)?;
1460 self.layers[layer_idx].cdat_root_tree(lex)
1461 }
1462
1463 pub(crate) fn graph_commit_meta(&self, oid: &ObjectId) -> Option<(ObjectId, Vec<ObjectId>)> {
1465 let (layer_idx, lex) = self.find_commit(oid)?;
1466 let tree = self.layers[layer_idx].cdat_root_tree(lex)?;
1467 let (parents, _) = self.graph_commit_at(layer_idx, lex)?;
1468 Some((tree, parents))
1469 }
1470
1471 pub(crate) fn graph_walk_entry(&self, oid: &ObjectId) -> Option<GraphWalkEntry> {
1474 let (layer_idx, lex) = self.find_commit(oid)?;
1475 let layer = &self.layers[layer_idx];
1476 let tree = layer.cdat_root_tree(lex)?;
1477 let (parents, time) = self.graph_commit_at(layer_idx, lex)?;
1478 Some(GraphWalkEntry {
1479 tree,
1480 parents,
1481 time,
1482 generation: layer
1483 .corrected_commit_generation(lex)
1484 .unwrap_or(UNRANKED_GENERATION),
1485 })
1486 }
1487
1488 #[must_use]
1490 pub fn commit_generation_for_walk(&self, oid: &ObjectId) -> u64 {
1491 let Some((layer_idx, lex)) = self.find_commit(oid) else {
1492 return UNRANKED_GENERATION;
1493 };
1494 self.layers[layer_idx]
1495 .corrected_commit_generation(lex)
1496 .unwrap_or(UNRANKED_GENERATION)
1497 }
1498
1499 #[must_use]
1502 pub fn parent_global_positions(&self, oid: &ObjectId) -> Option<Vec<u32>> {
1503 let (layer_idx, lex) = self.find_commit(oid)?;
1504 self.layers[layer_idx].parent_global_positions(lex)
1505 }
1506
1507 #[must_use]
1511 pub fn count_reachable_commits(
1512 &self,
1513 tips: &[ObjectId],
1514 hide: &[ObjectId],
1515 first_parent_only: bool,
1516 shallow: &std::collections::HashSet<ObjectId>,
1517 ) -> Option<usize> {
1518 if self.total_commits() == 0 {
1519 return None;
1520 }
1521 let mut seen = self.reachable_positions(tips, first_parent_only, shallow)?;
1522 if hide.is_empty() {
1523 return Some(seen.len());
1524 }
1525 let hidden = self.reachable_positions(hide, first_parent_only, shallow)?;
1526 seen.retain(|pos| !hidden.contains(pos));
1527 Some(seen.len())
1528 }
1529
1530 fn reachable_positions(
1531 &self,
1532 tips: &[ObjectId],
1533 first_parent_only: bool,
1534 shallow: &std::collections::HashSet<ObjectId>,
1535 ) -> Option<std::collections::HashSet<u32>> {
1536 let total = self.total_commits();
1537 let mut stack: Vec<u32> = Vec::new();
1538 for tip in tips {
1539 stack.push(self.global_position(tip)?);
1540 }
1541 let mut seen = std::collections::HashSet::new();
1542 while let Some(pos) = stack.pop() {
1543 if pos >= total || !seen.insert(pos) {
1544 continue;
1545 }
1546 #[cfg(test)]
1547 crate::merge_base::intersect_walk_test_hooks::record_commit_parent_lookup();
1548 let oid = self.oid_at_global_position(pos)?;
1549 let parents = if shallow.contains(&oid) {
1550 Vec::new()
1551 } else {
1552 self.parent_global_positions(&oid)?
1553 };
1554 let parents = if first_parent_only {
1555 parents.into_iter().take(1).collect::<Vec<_>>()
1556 } else {
1557 parents
1558 };
1559 for p in parents {
1560 if p >= total {
1561 return None;
1562 }
1563 stack.push(p);
1564 }
1565 }
1566 Some(seen)
1567 }
1568
1569 pub fn all_oids_in_order(&self) -> Vec<ObjectId> {
1571 let mut out = Vec::new();
1572 for layer in self.layers.iter().rev() {
1573 for i in 0..layer.num_commits {
1574 if let Some(oid) = layer.oid_at_lex(i) {
1575 out.push(oid);
1576 }
1577 }
1578 }
1579 out
1580 }
1581
1582 pub fn bloom_precheck_for_paths(
1584 &self,
1585 _odb: &Odb,
1586 oid: ObjectId,
1587 pathspecs: &[String],
1588 bloom_cwd: Option<&str>,
1589 requested_hash_version: i32,
1590 read_changed_paths: bool,
1591 ) -> std::result::Result<BloomPrecheck, crate::error::Error> {
1592 if !read_changed_paths {
1593 return Ok(BloomPrecheck::Inapplicable);
1594 }
1595 let Some((layer_idx, lex)) = self.find_commit(&oid) else {
1596 return Ok(BloomPrecheck::NotInGraph);
1597 };
1598 let layer = &self.layers[layer_idx];
1599 let Some(settings) = layer.bloom_settings.as_ref() else {
1600 return Ok(BloomPrecheck::FilterNotPresent);
1601 };
1602 let effective_version = if requested_hash_version < 0 {
1603 settings.hash_version as i32
1604 } else {
1605 requested_hash_version
1606 };
1607 if effective_version != settings.hash_version as i32 {
1608 return Ok(BloomPrecheck::FilterNotPresent);
1609 }
1610
1611 let filter = match layer.bloom_filter_slice(lex) {
1617 Some(s) => s,
1618 None => return Ok(BloomPrecheck::FilterNotPresent),
1619 };
1620 if filter.is_empty() {
1621 return Ok(BloomPrecheck::FilterNotPresent);
1622 }
1623
1624 let mut any_pathspec_maybe = false;
1627 let mut checked_any_keys = false;
1628 for spec in pathspecs {
1629 if spec.is_empty() || crate::pathspec::pathspec_is_exclude(spec) {
1630 continue;
1631 }
1632 let Some(norm) = crate::pathspec::bloom_lookup_prefix_with_cwd(spec, bloom_cwd) else {
1633 continue;
1634 };
1635 let keys = bloom_keyvec_for_path(norm.as_str(), settings);
1636 if keys.is_empty() {
1637 continue;
1638 }
1639 checked_any_keys = true;
1640 let mut all_keys_maybe = true;
1641 for key in &keys {
1642 match bloom_filter_contains(key, filter, settings) {
1643 Ok(true) => {}
1644 Ok(false) => {
1645 all_keys_maybe = false;
1646 break;
1647 }
1648 Err(crate::bloom::BloomFilterInvalid) => {
1649 all_keys_maybe = true;
1650 break;
1651 }
1652 }
1653 }
1654 if all_keys_maybe {
1655 any_pathspec_maybe = true;
1656 break;
1657 }
1658 }
1659 if checked_any_keys && !any_pathspec_maybe {
1660 return Ok(BloomPrecheck::DefinitelyNot);
1661 }
1662 Ok(BloomPrecheck::Maybe)
1663 }
1664
1665 pub(crate) fn bloom_definitely_not_for_keys(
1667 &self,
1668 oid: ObjectId,
1669 path_keys: &[Vec<u32>],
1670 ) -> bool {
1671 if path_keys.is_empty() {
1672 return false;
1673 }
1674 let Some((layer_idx, lex)) = self.find_commit(&oid) else {
1675 return false;
1676 };
1677 let layer = &self.layers[layer_idx];
1678 let Some(settings) = layer.bloom_settings.as_ref() else {
1679 return false;
1680 };
1681 let Some(filter) = layer.bloom_filter_slice(lex) else {
1682 return false;
1683 };
1684 if filter.is_empty() {
1685 return false;
1686 }
1687 bloom_definitely_not_from_filter(filter, settings, path_keys)
1688 }
1689}
1690
1691pub(crate) fn bloom_definitely_not_from_filter(
1693 filter: &[u8],
1694 settings: &crate::bloom::BloomFilterSettings,
1695 path_keys: &[Vec<u32>],
1696) -> bool {
1697 use crate::bloom::bloom_filter_contains;
1698
1699 for key in path_keys {
1700 match bloom_filter_contains(key, filter, settings) {
1701 Ok(false) => return true,
1702 Ok(true) => {}
1703 Err(crate::bloom::BloomFilterInvalid) => return false,
1704 }
1705 }
1706 false
1707}
1708
1709pub fn diff_changed_paths_for_bloom(
1711 odb: &Odb,
1712 parent_tree: Option<ObjectId>,
1713 commit_tree: ObjectId,
1714) -> crate::error::Result<(Vec<String>, usize)> {
1715 use crate::diff::diff_trees;
1716 let entries = diff_trees(odb, parent_tree.as_ref(), Some(&commit_tree), "")?;
1717 let raw_len = entries.len();
1718 let mut paths = Vec::new();
1719 for e in entries {
1720 let p = e.path().to_string();
1721 if !p.is_empty() {
1722 paths.push(p);
1723 }
1724 }
1725 Ok((paths, raw_len))
1726}
1727
1728pub use crate::bloom::collect_changed_paths_for_bloom;
1730
1731pub(crate) fn bloom_filter_for_commit_write(
1733 odb: &Odb,
1734 parents: &[ObjectId],
1735 tree_oid: ObjectId,
1736 settings: &BloomFilterSettings,
1737) -> crate::error::Result<(Vec<u8>, BloomBuildOutcome)> {
1738 let parent_tree = parents
1742 .first()
1743 .map(|first_parent| odb.commit_tree(first_parent))
1744 .transpose()?;
1745 bloom_filter_for_tree_diff(odb, parent_tree, tree_oid, settings)
1746}
1747
1748pub(crate) fn bloom_filter_for_tree_diff(
1751 odb: &Odb,
1752 parent_tree: Option<ObjectId>,
1753 tree_oid: ObjectId,
1754 settings: &BloomFilterSettings,
1755) -> crate::error::Result<(Vec<u8>, BloomBuildOutcome)> {
1756 let (changed_paths_vec, raw_count) = diff_changed_paths_for_bloom(odb, parent_tree, tree_oid)?;
1757 let set = collect_changed_paths_for_bloom(&changed_paths_vec);
1758 Ok(crate::bloom::build_bloom_filter_data(
1759 &set, raw_count, settings,
1760 ))
1761}
1762
1763pub(crate) fn tree_has_high_bit_paths(odb: &Odb, tree_oid: ObjectId) -> bool {
1768 let Ok(obj) = odb.read(&tree_oid) else {
1769 return true;
1771 };
1772 let Ok(entries) = crate::objects::parse_tree(&obj.data) else {
1773 return true;
1774 };
1775 for e in &entries {
1776 if e.name.iter().any(|&b| b & 0x80 != 0) {
1777 return true;
1778 }
1779 if e.mode == 0o040000 && tree_has_high_bit_paths(odb, e.oid) {
1780 return true;
1781 }
1782 }
1783 false
1784}
1785
1786pub(crate) fn commit_tree_has_high_bit_paths(odb: &Odb, commit_oid: ObjectId) -> bool {
1789 match odb.commit_tree(&commit_oid) {
1790 Ok(tree) => tree_has_high_bit_paths(odb, tree),
1791 Err(_) => true,
1792 }
1793}
1794
1795pub(crate) fn parse_graph_file(path: &Path) -> Option<ParsedGraphDump> {
1797 let raw = std::fs::read(path).ok()?;
1798 if raw.len() < 28 {
1799 return None;
1800 }
1801 let hash_len = commit_graph_hash_len(raw[5])?;
1802 let body = &raw[..raw.len() - hash_len];
1803 if body.len() < 8 || &body[0..4] != SIGNATURE {
1804 return None;
1805 }
1806 let header_word = u32::from_be_bytes(body[0..4].try_into().ok()?);
1807 let num_chunks = body[6] as usize;
1808 let toc_start = 8;
1809 let mut present: std::collections::HashSet<u32> = std::collections::HashSet::new();
1810 for i in 0..num_chunks {
1811 let e = toc_start + i * 12;
1812 let id = u32::from_be_bytes(body[e..e + 4].try_into().ok()?);
1813 present.insert(id);
1814 }
1815 let mut chunk_names: Vec<String> = Vec::new();
1818 for (id, label) in [
1819 (CHUNK_OID_FANOUT, "oid_fanout"),
1820 (CHUNK_OID_LOOKUP, "oid_lookup"),
1821 (CHUNK_COMMIT_DATA, "commit_metadata"),
1822 (CHUNK_GENERATION_DATA, "generation_data"),
1823 (CHUNK_GENERATION_DATA_OVERFLOW, "generation_data_overflow"),
1824 (CHUNK_EXTRA_EDGES, "extra_edges"),
1825 (CHUNK_BLOOM_INDEXES, "bloom_indexes"),
1826 (CHUNK_BLOOM_DATA, "bloom_data"),
1827 ] {
1828 if present.contains(&id) {
1829 chunk_names.push(label.to_string());
1830 }
1831 }
1832 let layer = CommitGraphLayer::parse(path.to_path_buf(), raw.clone())?;
1833 let bloom_opt = layer.bloom_settings.map(|s| {
1834 format!(
1835 " bloom({},{},{})",
1836 s.hash_version, s.bits_per_entry, s.num_hashes
1837 )
1838 });
1839 let mut options = String::new();
1840 if let Some(b) = bloom_opt {
1841 options.push_str(&b);
1842 }
1843 if layer.read_generation_data {
1844 options.push_str(" read_generation_data");
1845 }
1846 Some(ParsedGraphDump {
1847 header_word,
1848 version: body[4],
1849 hash_ver: body[5],
1850 num_chunks: body[6],
1851 reserved: body[7],
1852 num_commits: layer.num_commits,
1853 chunks: chunk_names.join(" "),
1854 options,
1855 })
1856}
1857
1858pub(crate) struct ParsedGraphDump {
1860 pub header_word: u32,
1862 pub version: u8,
1864 pub hash_ver: u8,
1866 pub num_chunks: u8,
1868 pub reserved: u8,
1870 pub num_commits: u32,
1872 pub chunks: String,
1874 pub options: String,
1876}
1877
1878pub(crate) fn dump_bloom_filters(path: &Path) -> Option<Vec<String>> {
1880 let raw = std::fs::read(path).ok()?;
1881 let layer = CommitGraphLayer::parse(path.to_path_buf(), raw)?;
1882 let mut out = Vec::new();
1883 for i in 0..layer.num_commits {
1884 let slice = layer.bloom_filter_slice(i).unwrap_or(&[]);
1885 if slice.is_empty() {
1886 out.push(String::new());
1887 } else {
1888 let hex: String = slice.iter().map(|b| format!("{b:02x}")).collect();
1889 out.push(hex);
1890 }
1891 }
1892 Some(out)
1893}
1894
1895#[cfg(test)]
1896mod tests {
1897 use std::collections::HashMap;
1898
1899 use crate::bloom::BloomFilterSettings;
1900 use crate::commit_graph_write::{build_commit_graph_bytes, load_commit_graph_commit_info};
1901 use crate::index::{Index, IndexEntry, MODE_REGULAR};
1902 use crate::objects::{serialize_commit, CommitData, ObjectId, ObjectKind};
1903 use crate::repo::{init_repository, Repository};
1904 use crate::write_tree::write_tree_from_index;
1905
1906 use crate::commit_graph_write::CommitGraphCommitInfo;
1907 use crate::objects::OidHashMap;
1908
1909 fn infos_std(
1910 infos: &OidHashMap<CommitGraphCommitInfo>,
1911 ) -> HashMap<ObjectId, CommitGraphCommitInfo> {
1912 infos.iter().map(|(k, v)| (*k, v.clone())).collect()
1913 }
1914
1915 fn empty_byte_filters() -> HashMap<ObjectId, Vec<u8>> {
1916 HashMap::new()
1917 }
1918
1919 fn one_commit_repo() -> (tempfile::TempDir, Repository, ObjectId) {
1920 let dir = tempfile::tempdir().expect("tempdir");
1921 let repo = init_repository(
1922 dir.path(),
1923 false,
1924 "main",
1925 None,
1926 crate::RefStorageFormat::Files,
1927 )
1928 .expect("init");
1929 let mut index = Index::new();
1930 index.hash_algo = repo.odb().hash_algo();
1931 let blob = repo.odb().write(ObjectKind::Blob, b"x\n").expect("blob");
1932 index.add_or_replace(IndexEntry {
1933 ctime_sec: 0,
1934 ctime_nsec: 0,
1935 mtime_sec: 0,
1936 mtime_nsec: 0,
1937 dev: 0,
1938 ino: 0,
1939 mode: MODE_REGULAR,
1940 uid: 0,
1941 gid: 0,
1942 size: 2,
1943 oid: blob,
1944 flags: 4,
1945 flags_extended: None,
1946 path: b"f.txt".to_vec(),
1947 base_index_pos: 0,
1948 });
1949 let tree = write_tree_from_index(&repo.odb(), &index, "").expect("tree");
1950 let raw = serialize_commit(&CommitData {
1951 tree,
1952 parents: vec![],
1953 author: "T <t@example.com> 100 +0000".into(),
1954 committer: "T <t@example.com> 100 +0000".into(),
1955 author_raw: Vec::new(),
1956 committer_raw: Vec::new(),
1957 encoding: None,
1958 message: "m\n".into(),
1959 raw_message: None,
1960 extra_headers: Vec::new(),
1961 });
1962 let commit = repo.odb().write(ObjectKind::Commit, &raw).expect("commit");
1963 crate::refs::write_ref(&repo.git_dir(), "refs/heads/main", &commit).expect("ref");
1964 (dir, repo, commit)
1965 }
1966
1967 fn reseal_graph(bytes: &mut Vec<u8>, algo: crate::objects::HashAlgo) {
1968 let hash_len = algo.len();
1969 let body_len = bytes.len().saturating_sub(hash_len);
1970 let digest = algo.checksum_digest(&bytes[..body_len]);
1971 bytes.truncate(body_len);
1972 bytes.extend_from_slice(&digest);
1973 }
1974
1975 fn memory_primary_over(
1976 dir: &tempfile::TempDir,
1977 files: &Repository,
1978 graph: Option<std::sync::Arc<super::CommitGraphChain>>,
1979 ) -> Repository {
1980 use crate::odb::store::{MemoryStore, ObjectStore, WritableObjectStore};
1981 let store = std::sync::Arc::new(MemoryStore::new(files.odb().hash_algo()));
1982 let source = files.odb().primary().expect("files primary");
1983 let mut oids = Vec::new();
1984 source
1985 .for_each_object(&mut |oid| {
1986 oids.push(*oid);
1987 std::ops::ControlFlow::Continue(())
1988 })
1989 .expect("list objects");
1990 for oid in oids {
1991 let obj = source.read(&oid).expect("read").expect("present");
1992 store
1993 .write(obj.kind, &obj.data, crate::odb::WriteOptions::default())
1994 .expect("copy");
1995 }
1996 let git_dir = dir.path().join(".git");
1997 let mut builder = crate::odb::OdbBuilder::files(git_dir.join("objects"))
1998 .primary(store)
1999 .alternates(false);
2000 if let Some(graph) = graph {
2001 builder = builder.commit_graph(graph);
2002 }
2003 crate::repo::RepositoryBuilder::new(crate::environment::RepositoryOptions::empty())
2004 .odb_builder(builder)
2005 .open(&git_dir, Some(dir.path()))
2006 .expect("open custom")
2007 }
2008
2009 #[test]
2010 fn from_bytes_parses_a_written_graph_and_rejects_bad_files() {
2011 let (_dir, repo, commit) = one_commit_repo();
2012 repo.write_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2013 .expect("write graph");
2014 let bytes =
2015 std::fs::read(repo.odb().objects_dir().join("info/commit-graph")).expect("graph file");
2016
2017 let chain = super::CommitGraphChain::from_bytes(bytes.clone()).expect("parse");
2018 assert_eq!(chain.total_commits(), 1);
2019 assert_eq!(chain.hash_algo(), crate::objects::HashAlgo::Sha1);
2020 assert_eq!(chain.graph_commit(&commit).map(|(p, _)| p), Some(vec![]));
2021
2022 let mut flipped = bytes.clone();
2023 let middle = flipped.len() / 2;
2024 flipped[middle] ^= 0xff;
2025 assert!(matches!(
2026 super::CommitGraphChain::from_bytes(flipped),
2027 Err(crate::error::Error::CorruptObject(_))
2028 ));
2029 assert!(super::CommitGraphChain::from_bytes(bytes[..20].to_vec()).is_err());
2030 assert!(super::CommitGraphChain::from_bytes(Vec::new()).is_err());
2031
2032 let mut split_layer = bytes;
2033 split_layer[7] = 1;
2034 reseal_graph(&mut split_layer, crate::objects::HashAlgo::Sha1);
2035 assert!(matches!(
2036 super::CommitGraphChain::from_bytes(split_layer),
2037 Err(crate::error::Error::CorruptObject(_))
2038 ));
2039 }
2040
2041 #[test]
2042 fn from_bytes_rejects_octopus_parents_without_an_edge_chunk() {
2043 let (_dir, repo, root) = one_commit_repo();
2044 let tree = repo.odb().read_commit(&root).expect("root commit").tree;
2045 let commit = |parents: Vec<ObjectId>, message: &str| {
2046 let raw = serialize_commit(&CommitData {
2047 tree,
2048 parents,
2049 author: "T <t@example.com> 200 +0000".into(),
2050 committer: "T <t@example.com> 200 +0000".into(),
2051 author_raw: Vec::new(),
2052 committer_raw: Vec::new(),
2053 encoding: None,
2054 message: message.into(),
2055 raw_message: None,
2056 extra_headers: Vec::new(),
2057 });
2058 repo.odb().write(ObjectKind::Commit, &raw).expect("commit")
2059 };
2060 let side_a = commit(vec![], "a\n");
2061 let side_b = commit(vec![], "b\n");
2062 let octopus = commit(vec![root, side_a, side_b], "octopus\n");
2063 crate::refs::write_ref(&repo.git_dir(), "refs/heads/main", &octopus).expect("ref");
2064 repo.write_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2065 .expect("write graph");
2066 let bytes =
2067 std::fs::read(repo.odb().objects_dir().join("info/commit-graph")).expect("graph file");
2068 let chain = super::CommitGraphChain::from_bytes(bytes.clone()).expect("valid graph");
2069 assert_eq!(chain.total_commits(), 4);
2070 assert_eq!(
2071 chain
2072 .graph_commit(&octopus)
2073 .map(|(parents, _)| parents.len()),
2074 Some(3)
2075 );
2076
2077 let mut no_edge_chunk = bytes;
2078 let edge_entry = (0..no_edge_chunk[6] as usize)
2079 .map(|i| 8 + i * 12)
2080 .find(|&e| no_edge_chunk[e..e + 4] == super::CHUNK_EXTRA_EDGES.to_be_bytes())
2081 .expect("octopus graph has an EDGE chunk");
2082 no_edge_chunk[edge_entry..edge_entry + 4].copy_from_slice(b"EDGX");
2083 reseal_graph(&mut no_edge_chunk, crate::objects::HashAlgo::Sha1);
2084 assert!(matches!(
2085 super::CommitGraphChain::from_bytes(no_edge_chunk),
2086 Err(crate::error::Error::CorruptObject(_))
2087 ));
2088 }
2089
2090 #[test]
2091 fn custom_primary_ignores_graph_files_and_uses_the_attached_graph() {
2092 let (dir, files, commit) = one_commit_repo();
2093 files
2094 .write_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2095 .expect("write graph");
2096 assert!(files.odb().commit_graph_chain().is_some());
2097
2098 let custom = memory_primary_over(&dir, &files, None);
2099 assert!(
2100 custom.odb().commit_graph_chain().is_none(),
2101 "a graph file under objects/ does not describe a custom store"
2102 );
2103
2104 let crate::maintenance::CommitGraphBuild::Built(built) = custom
2105 .build_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2106 .expect("build")
2107 else {
2108 panic!("plain repository must not skip the graph");
2109 };
2110 assert_eq!(built.commit_count, 1);
2111 assert_eq!(built.bloom_stats, None);
2112 assert_eq!(
2113 built.bytes,
2114 std::fs::read(files.odb().objects_dir().join("info/commit-graph")).expect("file"),
2115 "the in-memory build matches the files writer byte for byte"
2116 );
2117
2118 let chain = std::sync::Arc::new(built.to_chain().expect("chain"));
2119 let attached = memory_primary_over(&dir, &files, Some(std::sync::Arc::clone(&chain)));
2120 let used = attached.odb().commit_graph_chain().expect("attached graph");
2121 assert!(std::sync::Arc::ptr_eq(&used, &chain));
2122 assert!(used.graph_commit(&commit).is_some());
2123 }
2124
2125 #[test]
2126 fn empty_commit_graph_bytes_parse_and_open() {
2127 let dir = tempfile::tempdir().expect("tempdir");
2128 let repo = init_repository(
2129 dir.path(),
2130 false,
2131 "main",
2132 None,
2133 crate::RefStorageFormat::Files,
2134 )
2135 .expect("init");
2136 let bloom = BloomFilterSettings::default();
2137 let (bytes, _) = build_commit_graph_bytes(
2138 &[],
2139 &HashMap::new(),
2140 &repo.odb(),
2141 false,
2142 &bloom,
2143 None,
2144 &[],
2145 None,
2146 &empty_byte_filters(),
2147 &empty_byte_filters(),
2148 true,
2149 )
2150 .expect("build");
2151 let layer = super::CommitGraphLayer::try_parse(
2152 std::path::PathBuf::from("commit-graph"),
2153 bytes.clone(),
2154 )
2155 .expect("parse");
2156 assert_eq!(layer.num_commits, 0);
2157 let path = dir.path().join("commit-graph");
2158 std::fs::write(&path, &bytes).expect("write");
2159 let opened = super::CommitGraphLayer::try_open(path).expect("open");
2160 assert_eq!(opened.num_commits, 0);
2161 }
2162
2163 #[test]
2164 fn try_parse_rejects_bad_signature_and_truncated_toc() {
2165 let (_dir, repo, commit) = one_commit_repo();
2166 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2167 let mut infos = crate::objects::oid_hash_map();
2168 infos.insert(commit, info);
2169 let bloom = BloomFilterSettings::default();
2170 let (mut bytes, _) = build_commit_graph_bytes(
2171 &[commit],
2172 &infos_std(&infos),
2173 &repo.odb(),
2174 false,
2175 &bloom,
2176 None,
2177 &[],
2178 None,
2179 &empty_byte_filters(),
2180 &empty_byte_filters(),
2181 true,
2182 )
2183 .expect("write");
2184 let algo = repo.odb().hash_algo();
2185 bytes[0] = 0;
2186 reseal_graph(&mut bytes, algo);
2187 let err = super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes.clone())
2188 .unwrap_err();
2189 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2190 bytes.truncate(40);
2191 let err =
2192 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes).unwrap_err();
2193 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2194 let (mut good, _) = build_commit_graph_bytes(
2195 &[commit],
2196 &infos_std(&infos),
2197 &repo.odb(),
2198 false,
2199 &bloom,
2200 None,
2201 &[],
2202 None,
2203 &empty_byte_filters(),
2204 &empty_byte_filters(),
2205 true,
2206 )
2207 .expect("write");
2208 good[4] = 2;
2209 reseal_graph(&mut good, algo);
2210 let err =
2211 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), good).unwrap_err();
2212 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2213 let (mut checksum_bad, _) = build_commit_graph_bytes(
2214 &[commit],
2215 &infos_std(&infos),
2216 &repo.odb(),
2217 false,
2218 &bloom,
2219 None,
2220 &[],
2221 None,
2222 &empty_byte_filters(),
2223 &empty_byte_filters(),
2224 true,
2225 )
2226 .expect("write");
2227 let last = checksum_bad.len() - 1;
2228 checksum_bad[last] ^= 0xff;
2229 let err = super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), checksum_bad)
2230 .unwrap_err();
2231 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2232 }
2233
2234 #[test]
2235 fn parse_graph_file_returns_none_for_invalid_inputs() {
2236 let dir = tempfile::tempdir().expect("tempdir");
2237 let missing = dir.path().join("nope.graph");
2238 assert!(super::parse_graph_file(&missing).is_none());
2239 std::fs::write(dir.path().join("tiny.graph"), b"CGPH").expect("write");
2240 assert!(super::parse_graph_file(&dir.path().join("tiny.graph")).is_none());
2241 assert!(super::dump_bloom_filters(&missing).is_none());
2242 }
2243
2244 #[test]
2245 fn try_load_missing_graph_is_none() {
2246 let dir = tempfile::tempdir().expect("tempdir");
2247 let objects = dir.path().join("objects");
2248 std::fs::create_dir_all(objects.join("info")).expect("info");
2249 assert!(super::CommitGraphChain::try_load(&objects)
2250 .expect("ok")
2251 .is_none());
2252 }
2253
2254 #[test]
2255 fn try_load_across_finds_single_graph() {
2256 let (_dir, repo, _commit) = one_commit_repo();
2257 let objects = repo.odb().objects_dir();
2258 let sorted = vec![_commit];
2259 let info = load_commit_graph_commit_info(&repo.odb(), _commit).expect("info");
2260 let mut infos = crate::objects::oid_hash_map();
2261 infos.insert(_commit, info);
2262 let bloom = BloomFilterSettings::default();
2263 let (bytes, _) = build_commit_graph_bytes(
2264 &sorted,
2265 &infos_std(&infos),
2266 &repo.odb(),
2267 false,
2268 &bloom,
2269 None,
2270 &[],
2271 None,
2272 &empty_byte_filters(),
2273 &empty_byte_filters(),
2274 true,
2275 )
2276 .expect("write");
2277 std::fs::write(objects.join("info/commit-graph"), bytes).expect("write");
2278 let chain = super::CommitGraphChain::try_load_across(objects, &[])
2279 .expect("across")
2280 .expect("some");
2281 assert_eq!(chain.total_commits(), 1);
2282 }
2283
2284 #[test]
2285 fn try_parse_rejects_oidf_toc_offset_out_of_range() {
2286 let (_dir, repo, commit) = one_commit_repo();
2287 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2288 let mut infos = crate::objects::oid_hash_map();
2289 infos.insert(commit, info);
2290 let bloom = BloomFilterSettings::default();
2291 let (mut bytes, _) = build_commit_graph_bytes(
2292 &[commit],
2293 &infos_std(&infos),
2294 &repo.odb(),
2295 false,
2296 &bloom,
2297 None,
2298 &[],
2299 None,
2300 &empty_byte_filters(),
2301 &empty_byte_filters(),
2302 true,
2303 )
2304 .expect("write");
2305 let algo = repo.odb().hash_algo();
2306 let num_chunks = bytes[6] as usize;
2307 for i in 0..num_chunks {
2308 let e = 8 + i * 12;
2309 let id = u32::from_be_bytes(bytes[e..e + 4].try_into().unwrap());
2310 if id == 0x4f49_4446 {
2311 bytes[e + 4..e + 12].copy_from_slice(&u64::MAX.to_be_bytes());
2312 break;
2313 }
2314 }
2315 reseal_graph(&mut bytes, algo);
2316 let err =
2317 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes).unwrap_err();
2318 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2319 }
2320
2321 #[test]
2322 fn generation_overflow_without_gdo2_fails_parse() {
2323 let (_dir, repo, commit) = one_commit_repo();
2324 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2325 let mut infos = crate::objects::oid_hash_map();
2326 infos.insert(commit, info);
2327 let bloom = BloomFilterSettings::default();
2328 let (mut bytes, _) = build_commit_graph_bytes(
2329 &[commit],
2330 &infos_std(&infos),
2331 &repo.odb(),
2332 false,
2333 &bloom,
2334 None,
2335 &[],
2336 None,
2337 &empty_byte_filters(),
2338 &empty_byte_filters(),
2339 true,
2340 )
2341 .expect("write");
2342 if bytes.len() > 100 {
2343 for i in 80..120 {
2344 if i + 4 <= bytes.len() - 20 {
2345 bytes[i] = 0x80;
2346 bytes[i + 1] = 0x00;
2347 bytes[i + 2] = 0x00;
2348 bytes[i + 3] = 0x00;
2349 break;
2350 }
2351 }
2352 }
2353 let err =
2354 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes).unwrap_err();
2355 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2356 }
2357
2358 #[test]
2359 fn bloom_precheck_not_in_graph_for_missing_oid() {
2360 let (_dir, repo, commit) = one_commit_repo();
2361 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2362 let mut infos = crate::objects::oid_hash_map();
2363 infos.insert(commit, info);
2364 let bloom = BloomFilterSettings::default();
2365 let (bytes, _) = build_commit_graph_bytes(
2366 &[commit],
2367 &infos_std(&infos),
2368 &repo.odb(),
2369 true,
2370 &bloom,
2371 None,
2372 &[],
2373 None,
2374 &empty_byte_filters(),
2375 &empty_byte_filters(),
2376 true,
2377 )
2378 .expect("write");
2379 std::fs::write(repo.odb().objects_dir().join("info/commit-graph"), bytes).expect("write");
2380 let chain = super::CommitGraphChain::load(repo.odb().objects_dir()).expect("chain");
2381 let missing = ObjectId::from_hex("1234567890123456789012345678901234567890").expect("oid");
2382 let pre = chain
2383 .bloom_precheck_for_paths(&repo.odb(), missing, &[], None, -1, true)
2384 .expect("precheck");
2385 assert_eq!(pre, super::BloomPrecheck::NotInGraph);
2386 }
2387
2388 #[test]
2389 fn bloom_precheck_respects_read_changed_paths_flag() {
2390 let (_dir, repo, commit) = one_commit_repo();
2391 let mut sorted = vec![commit];
2392 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2393 let mut infos = crate::objects::oid_hash_map();
2394 infos.insert(commit, info);
2395 let bloom = BloomFilterSettings::default();
2396 let (bytes, _) = build_commit_graph_bytes(
2397 &sorted,
2398 &infos_std(&infos),
2399 &repo.odb(),
2400 true,
2401 &bloom,
2402 None,
2403 &[],
2404 None,
2405 &empty_byte_filters(),
2406 &empty_byte_filters(),
2407 true,
2408 )
2409 .expect("write");
2410 std::fs::write(repo.odb().objects_dir().join("info/commit-graph"), bytes).expect("write");
2411 let chain = super::CommitGraphChain::load(repo.odb().objects_dir()).expect("chain");
2412 let pre = chain
2413 .bloom_precheck_for_paths(&repo.odb(), commit, &["f.txt".to_owned()], None, -1, false)
2414 .expect("precheck");
2415 assert_eq!(pre, super::BloomPrecheck::Inapplicable);
2416 sorted.clear();
2417 }
2418
2419 #[test]
2420 fn bloom_skips_most_steps_when_tracked_path_unchanged_in_linear_history() {
2421 use crate::bloom::bloom_keyvec_for_path;
2422 use crate::commit_graph_write::{build_commit_graph_bytes, load_commit_graph_commit_info};
2423 use crate::index::{Index, IndexEntry, MODE_REGULAR};
2424 use crate::objects::{serialize_commit, CommitData, ObjectKind};
2425 use crate::pathspec::bloom_lookup_prefix_with_cwd;
2426 use crate::repo::init_repository;
2427 use crate::write_tree::write_tree_from_index;
2428 use crate::RefStorageFormat;
2429 use std::fs;
2430
2431 let dir = tempfile::tempdir().expect("tempdir");
2432 let repo = init_repository(dir.path(), false, "main", None, RefStorageFormat::Files)
2433 .expect("init");
2434 let path = "tracked.txt";
2435 let stable = b"stable payload\n";
2436 let blob = repo.odb().write(ObjectKind::Blob, stable).expect("blob");
2437 let mut parent: Option<ObjectId> = None;
2438 let mut oids = Vec::new();
2439 for i in 0..24usize {
2440 let mut index = Index::new();
2441 index.hash_algo = repo.odb().hash_algo();
2442 index.add_or_replace(IndexEntry {
2443 ctime_sec: 0,
2444 ctime_nsec: 0,
2445 mtime_sec: 0,
2446 mtime_nsec: 0,
2447 dev: 0,
2448 ino: 0,
2449 mode: MODE_REGULAR,
2450 uid: 0,
2451 gid: 0,
2452 size: stable.len() as u32,
2453 oid: blob,
2454 flags: 4,
2455 flags_extended: None,
2456 path: path.as_bytes().to_vec(),
2457 base_index_pos: 0,
2458 });
2459 if i % 5 == 3 {
2460 let noise = repo
2461 .odb()
2462 .write(ObjectKind::Blob, format!("noise {i}\n").as_bytes())
2463 .expect("noise blob");
2464 index.add_or_replace(IndexEntry {
2465 ctime_sec: 0,
2466 ctime_nsec: 0,
2467 mtime_sec: 0,
2468 mtime_nsec: 0,
2469 dev: 0,
2470 ino: 0,
2471 mode: MODE_REGULAR,
2472 uid: 0,
2473 gid: 0,
2474 size: 8,
2475 oid: noise,
2476 flags: 4,
2477 flags_extended: None,
2478 path: format!("other{i}.txt").into_bytes(),
2479 base_index_pos: 0,
2480 });
2481 }
2482 let tree = write_tree_from_index(&repo.odb(), &index, "").expect("tree");
2483 let mut parents = Vec::new();
2484 if let Some(p) = parent {
2485 parents.push(p);
2486 }
2487 let raw = serialize_commit(&CommitData {
2488 tree,
2489 parents,
2490 author: "T <t@example.com> 100 +0000".into(),
2491 committer: "T <t@example.com> 100 +0000".into(),
2492 author_raw: Vec::new(),
2493 committer_raw: Vec::new(),
2494 encoding: None,
2495 message: format!("c{i}\n"),
2496 raw_message: None,
2497 extra_headers: Vec::new(),
2498 });
2499 let commit = repo.odb().write(ObjectKind::Commit, &raw).expect("commit");
2500 oids.push(commit);
2501 parent = Some(commit);
2502 }
2503 let head = *oids.last().expect("head");
2504 let mut infos = crate::objects::oid_hash_map();
2505 for oid in &oids {
2506 infos.insert(
2507 *oid,
2508 load_commit_graph_commit_info(&repo.odb(), *oid).expect("info"),
2509 );
2510 }
2511 let bloom = BloomFilterSettings::default();
2512 let (bytes, _) = build_commit_graph_bytes(
2513 &oids,
2514 &infos_std(&infos),
2515 &repo.odb(),
2516 true,
2517 &bloom,
2518 None,
2519 &[],
2520 None,
2521 &empty_byte_filters(),
2522 &empty_byte_filters(),
2523 true,
2524 )
2525 .expect("graph bytes");
2526 fs::write(repo.odb().objects_dir().join("info/commit-graph"), bytes).expect("write graph");
2527 let chain = super::CommitGraphChain::load(repo.odb().objects_dir()).expect("chain");
2528 let settings = chain.top_layer_bloom_settings().expect("bloom");
2529 let norm = bloom_lookup_prefix_with_cwd(path, None).expect("prefix");
2530 let keys = bloom_keyvec_for_path(norm.as_str(), &settings);
2531 let mut current = head;
2532 let mut bloom_yes = 0usize;
2533 let mut bloom_no = 0usize;
2534 while bloom_yes + bloom_no < 10_000 {
2535 let Some((parents, _)) = chain.graph_commit(¤t) else {
2536 break;
2537 };
2538 let Some(parent) = parents.first().copied() else {
2539 break;
2540 };
2541 if chain.bloom_definitely_not_for_keys(current, keys.as_slice()) {
2542 bloom_yes += 1;
2543 } else {
2544 bloom_no += 1;
2545 }
2546 current = parent;
2547 }
2548 assert!(
2549 bloom_yes > bloom_no,
2550 "expected bloom to skip unchanged tracked path (yes={bloom_yes} no={bloom_no})"
2551 );
2552 }
2553
2554 #[test]
2555 #[ignore = "local git.git worktree; GRIT_BLAME_BENCH_REPO"]
2556 fn first_parent_makefile_bloom_skip_rate_on_git_dot_git() {
2557 use crate::bloom::bloom_keyvec_for_path;
2558 use crate::environment::RepositoryOptions;
2559 use crate::pathspec::bloom_lookup_prefix_with_cwd;
2560 use crate::repo::Repository;
2561
2562 let repo_path = std::env::var("GRIT_BLAME_BENCH_REPO")
2563 .unwrap_or_else(|_| "/tmp/grit-bench/git-dot-git-worktree".to_string());
2564 if !std::path::Path::new(&repo_path).join(".git").exists() {
2565 return;
2566 }
2567 let git_dir = std::path::Path::new(&repo_path).join(".git");
2568 let repo =
2569 Repository::open(git_dir.as_path(), &RepositoryOptions::default()).expect("open");
2570 let chain = repo.odb().commit_graph_chain().expect("graph");
2571 let settings = chain.top_layer_bloom_settings().expect("bloom");
2572 let norm = bloom_lookup_prefix_with_cwd("Makefile", None).expect("prefix");
2573 let keys = bloom_keyvec_for_path(norm.as_str(), &settings);
2574 let head = *crate::state::resolve_head(repo.git_dir())
2575 .expect("head")
2576 .oid()
2577 .expect("commit");
2578 let mut current = head;
2579 let mut bloom_yes = 0usize;
2580 let mut bloom_no = 0usize;
2581 for _ in 0..10_000 {
2582 let Some((parents, _)) = chain.graph_commit(¤t) else {
2583 break;
2584 };
2585 let Some(parent) = parents.first().copied() else {
2586 break;
2587 };
2588 if chain.bloom_definitely_not_for_keys(current, keys.as_slice()) {
2589 bloom_yes += 1;
2590 } else {
2591 bloom_no += 1;
2592 }
2593 current = parent;
2594 }
2595 assert!(
2596 bloom_yes > bloom_no,
2597 "expected mostly bloom skips, yes={bloom_yes} no={bloom_no}"
2598 );
2599 }
2600
2601 #[test]
2602 fn bloom_definitely_not_when_directory_key_present_but_not_full_path() {
2603 use crate::bloom::{
2604 bloom_filter_contains, bloom_keyvec_for_path, build_bloom_filter_data,
2605 collect_changed_paths_for_bloom,
2606 };
2607
2608 let bloom = BloomFilterSettings::default();
2609 let changed = collect_changed_paths_for_bloom(&["builtin/sibling.c".to_owned()]);
2610 let (filter, _) = build_bloom_filter_data(&changed, 1, &bloom);
2611 let path_keys = bloom_keyvec_for_path("builtin/commit.c", &bloom);
2612 assert!(path_keys.len() >= 2);
2613 assert_eq!(
2614 bloom_filter_contains(&path_keys[0], &filter, &bloom),
2615 Ok(false),
2616 "full path key must be absent when only a sibling changed"
2617 );
2618 assert_eq!(
2619 bloom_filter_contains(&path_keys[1], &filter, &bloom),
2620 Ok(true),
2621 "directory prefix key should be present"
2622 );
2623 assert!(super::bloom_definitely_not_from_filter(
2624 &filter, &bloom, &path_keys
2625 ));
2626 }
2627
2628 #[cfg_attr(
2629 not(feature = "full-tests"),
2630 ignore = "starts system git or is heavy; run with --features full-tests"
2631 )]
2632 #[test]
2633 fn large_git_split_chain_layers_use_mapped_backing() {
2634 use std::process::Command;
2635
2636 let git_ok = Command::new("git")
2637 .arg("--version")
2638 .output()
2639 .map(|o| o.status.success())
2640 .unwrap_or(false);
2641 if !git_ok {
2642 eprintln!("SKIP: git unavailable");
2643 return;
2644 }
2645 let dir = tempfile::tempdir().expect("tempdir");
2646 let run = |args: &[&str]| {
2647 let out = Command::new("git")
2648 .current_dir(dir.path())
2649 .args(args)
2650 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2651 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2652 .env("GIT_AUTHOR_NAME", "T")
2653 .env("GIT_AUTHOR_EMAIL", "t@example.com")
2654 .env("GIT_COMMITTER_NAME", "T")
2655 .env("GIT_COMMITTER_EMAIL", "t@example.com")
2656 .output()
2657 .expect("git");
2658 assert!(out.status.success(), "git {args:?}");
2659 };
2660 run(&["init", "-q", "-b", "main"]);
2661 run(&["commit", "--allow-empty", "-q", "-m", "bootstrap"]);
2662 for i in 0..150 {
2663 run(&[
2664 "commit",
2665 "--allow-empty",
2666 "-q",
2667 "-m",
2668 &format!("base-fill-{i}"),
2669 ]);
2670 }
2671 run(&[
2672 "commit-graph",
2673 "write",
2674 "--reachable",
2675 "--split",
2676 "--max-commits",
2677 "40",
2678 ]);
2679 for i in 150..190 {
2680 run(&[
2681 "commit",
2682 "--allow-empty",
2683 "-q",
2684 "-m",
2685 &format!("tip-fill-{i}"),
2686 ]);
2687 }
2688 run(&[
2689 "commit-graph",
2690 "write",
2691 "--reachable",
2692 "--split",
2693 "--max-commits",
2694 "40",
2695 ]);
2696 let objects = dir.path().join(".git/objects");
2697 let chain = super::CommitGraphChain::try_load(&objects)
2698 .expect("load")
2699 .expect("chain");
2700 assert!(
2701 chain.num_layers() >= 2,
2702 "expected multi-layer chain, got {}",
2703 chain.num_layers()
2704 );
2705 let mut saw_mapped_large_layer = false;
2706 for path in chain.layer_paths_oldest_first() {
2707 let len = std::fs::metadata(&path).expect("meta").len();
2708 if len <= 8192 {
2709 continue;
2710 }
2711 let storage = crate::pack_map::PackData::open_with_min_len(
2712 &path,
2713 super::COMMIT_GRAPH_MIN_FILE_LEN,
2714 )
2715 .expect("open layer");
2716 assert!(
2717 storage.is_mapped_backing(),
2718 "expected mmap backing for {} ({} bytes)",
2719 path.display(),
2720 len
2721 );
2722 saw_mapped_large_layer = true;
2723 }
2724 assert!(
2725 saw_mapped_large_layer,
2726 "expected at least one chain layer larger than 8192 bytes to use mmap"
2727 );
2728 }
2729}