1use std::path::{Path, PathBuf};
4use std::sync::{Arc, Mutex};
5
6use crate::bloom::{
7 bloom_filter_contains, bloom_keyvec_for_path, BloomBuildOutcome, BloomFilterSettings,
8};
9use crate::error::Error;
10use crate::hash::verify_trailer;
11use crate::objects::{HashAlgo, ObjectId};
12use crate::odb::{hash_algo_for_objects_dir, Odb};
13use crate::pack_map::PackData;
14
15fn warn_once_for_disabled_bloom_layer(
16 caches: Option<&crate::repo_caches::RepoCaches>,
17 id: &str,
18) -> bool {
19 let key = format!("bloom-disabled:{id}");
20 caches
21 .map(|c| c.should_warn_commit_graph_once(&key))
22 .unwrap_or(true)
23}
24
25fn warn_once_for_base_chunk_too_small(
26 caches: Option<&crate::repo_caches::RepoCaches>,
27 id: &str,
28) -> bool {
29 let key = format!("base-chunk:{id}");
30 caches
31 .map(|c| c.should_warn_commit_graph_once(&key))
32 .unwrap_or(true)
33}
34
35const SIGNATURE: &[u8; 4] = b"CGPH";
36const GRAPH_VERSION: u8 = 1;
37const COMMIT_GRAPH_MIN_FILE_LEN: u64 = 28;
38
39const CHUNK_OID_FANOUT: u32 = 0x4f49_4446; const CHUNK_OID_LOOKUP: u32 = 0x4f49_444c; const CHUNK_COMMIT_DATA: u32 = 0x4344_4154; const CHUNK_GENERATION_DATA: u32 = 0x4744_4132; const CHUNK_GENERATION_DATA_OVERFLOW: u32 = 0x4744_4f32; const CHUNK_EXTRA_EDGES: u32 = 0x4544_4745; const GRAPH_PARENT_NONE: u32 = 0x7000_0000;
48const GRAPH_EXTRA_EDGES_NEEDED: u32 = 0x8000_0000;
51const GRAPH_LAST_EDGE: u32 = 0x8000_0000;
52const CHUNK_BLOOM_INDEXES: u32 = 0x4249_4458; const CHUNK_BLOOM_DATA: u32 = 0x4244_4154; const CHUNK_BASE_GRAPHS: u32 = 0x4241_5345; const BLOOM_HEADER: usize = crate::bloom::BLOOMDATA_HEADER_LEN;
57
58const CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW: u32 = 1u32 << 31;
60
61fn warn_path_for_graph_file(path: &Path) -> String {
62 let s = path.to_string_lossy();
63 if let Some(idx) = s.find(".git/") {
64 return s[idx..].replace('\\', "/");
65 }
66 s.replace('\\', "/")
67}
68
69#[derive(Debug, Clone)]
71pub struct CommitGraphLayer {
72 pub path: PathBuf,
74 file: Arc<PackData>,
75 body_len: usize,
76 num_commits: u32,
77 oid_lookup_off: usize,
78 chunk_commit_data_off: usize,
79 chunk_generation_data: Option<usize>,
80 chunk_generation_overflow: Option<usize>,
81 read_generation_data: bool,
82 chunk_bloom_indexes: Option<usize>,
83 chunk_bloom_data: Option<(usize, usize)>,
84 bloom_settings: Option<BloomFilterSettings>,
85 bloom_disabled: bool,
86 base_chunk_size: usize,
89 hash_len: usize,
91 algo: HashAlgo,
93 extra_edges_off: Option<usize>,
94 extra_edges_end: Option<usize>,
95}
96
97fn commit_graph_hash_len(hash_version: u8) -> Option<usize> {
99 HashAlgo::try_from(hash_version).ok().map(|a| a.len())
100}
101
102fn commit_graph_chain_hash_is_valid(h: &str, algo: HashAlgo) -> bool {
104 h.len() == algo.hex_len() && h.chars().all(|c| c.is_ascii_hexdigit())
105}
106
107const OIDF_BUCKET_BYTES: usize = 256 * 4;
108
109fn commit_graph_chunk_oob() -> Error {
110 Error::CorruptObject("commit-graph chunk extends past end of file".to_owned())
111}
112
113fn commit_graph_offset_out_of_range() -> Error {
114 Error::CorruptObject("commit-graph chunk offset out of range".to_owned())
115}
116
117fn checked_add_end(start: usize, len: usize, bound: usize) -> Result<usize, Error> {
118 start
119 .checked_add(len)
120 .filter(|&end| end <= bound)
121 .ok_or_else(commit_graph_chunk_oob)
122}
123
124fn body_slice(body: &[u8], start: usize, len: usize) -> Result<&[u8], Error> {
125 let end = checked_add_end(start, len, body.len())?;
126 body.get(start..end).ok_or_else(commit_graph_chunk_oob)
127}
128
129fn parse_toc_offset(body_len: usize, raw: [u8; 8]) -> Result<usize, Error> {
130 let off = u64::from_be_bytes(raw);
131 if off > body_len as u64 {
132 return Err(commit_graph_offset_out_of_range());
133 }
134 usize::try_from(off).map_err(|_| commit_graph_offset_out_of_range())
135}
136
137fn fanout_num_commits(body: &[u8], fanout_off: usize) -> Result<u32, Error> {
138 let fanout = body_slice(body, fanout_off, OIDF_BUCKET_BYTES)?;
139 let word = fanout[255 * 4..256 * 4]
140 .try_into()
141 .map_err(|_| Error::CorruptObject("commit-graph fanout corrupt".to_owned()))?;
142 Ok(u32::from_be_bytes(word))
143}
144
145#[allow(clippy::too_many_arguments)]
146fn validate_single_layer_parent_indices(
147 body: &[u8],
148 num_commits: u32,
149 commit_data_off: usize,
150 graph_data_width: usize,
151 hash_len: usize,
152 extra_edges_off: Option<usize>,
153 toc_entries: &[(u32, usize)],
154 file_end: usize,
155) -> Result<(), Error> {
156 fn chunk_byte_range(
157 start: usize,
158 toc_entries: &[(u32, usize)],
159 file_end: usize,
160 ) -> Result<usize, Error> {
161 let mut ends: Vec<usize> = toc_entries
162 .iter()
163 .map(|&(_, o)| o)
164 .filter(|&o| o > start)
165 .collect();
166 ends.sort_unstable();
167 Ok(ends.first().copied().unwrap_or(file_end))
168 }
169
170 for lex in 0..num_commits {
171 let base = commit_data_off + lex as usize * graph_data_width;
172 let p1_off = base + hash_len;
173 let p1 =
174 u32::from_be_bytes(body[p1_off..p1_off + 4].try_into().map_err(|_| {
175 Error::CorruptObject("commit-graph commit data corrupt".to_owned())
176 })?);
177 let p2 =
178 u32::from_be_bytes(body[p1_off + 4..p1_off + 8].try_into().map_err(|_| {
179 Error::CorruptObject("commit-graph commit data corrupt".to_owned())
180 })?);
181 if p1 != GRAPH_PARENT_NONE
182 && p1 & GRAPH_EXTRA_EDGES_NEEDED == 0
183 && p1 as usize >= num_commits as usize
184 {
185 return Err(Error::CorruptObject(
186 "commit-graph invalid parent index".to_owned(),
187 ));
188 }
189 if p2 != GRAPH_PARENT_NONE {
190 if p2 & GRAPH_EXTRA_EDGES_NEEDED != 0 {
191 let start = (p2 & !GRAPH_EXTRA_EDGES_NEEDED) as usize;
192 let Some(edge_off) = extra_edges_off else {
193 return Err(Error::CorruptObject(
194 "commit-graph extra edge list out of range".to_owned(),
195 ));
196 };
197 let edge_end = chunk_byte_range(edge_off, toc_entries, file_end)?;
198 let edge_len = edge_end.saturating_sub(edge_off);
199 if start * 4 >= edge_len {
200 return Err(Error::CorruptObject(
201 "commit-graph extra edge list out of range".to_owned(),
202 ));
203 }
204 let mut idx = start;
205 loop {
206 if idx * 4 + 4 > edge_len {
207 return Err(Error::CorruptObject(
208 "commit-graph extra edge list corrupt".to_owned(),
209 ));
210 }
211 let ev = u32::from_be_bytes(
212 body[edge_off + idx * 4..edge_off + idx * 4 + 4]
213 .try_into()
214 .map_err(|_| {
215 Error::CorruptObject(
216 "commit-graph extra edge list corrupt".to_owned(),
217 )
218 })?,
219 );
220 let parent = ev & !GRAPH_LAST_EDGE;
221 if parent != GRAPH_PARENT_NONE
222 && parent & GRAPH_EXTRA_EDGES_NEEDED == 0
223 && parent as usize >= num_commits as usize
224 {
225 return Err(Error::CorruptObject(
226 "commit-graph invalid parent index".to_owned(),
227 ));
228 }
229 if ev & GRAPH_LAST_EDGE != 0 {
230 break;
231 }
232 idx += 1;
233 }
234 } else if p2 as usize >= num_commits as usize {
235 return Err(Error::CorruptObject(
236 "commit-graph invalid parent index".to_owned(),
237 ));
238 }
239 }
240 }
241 Ok(())
242}
243
244#[derive(Clone, Copy, PartialEq, Eq)]
245enum CommitGraphParseMode {
246 Open,
248 Verify,
250}
251
252impl CommitGraphLayer {
253 fn body(&self) -> &[u8] {
254 &self.file[..self.body_len]
255 }
256
257 fn try_open(path: PathBuf) -> Result<Self, Error> {
259 let storage = PackData::open_with_min_len(&path, COMMIT_GRAPH_MIN_FILE_LEN)?;
260 Self::try_parse_storage(path, storage, CommitGraphParseMode::Open)
261 }
262
263 pub fn try_parse(path: PathBuf, raw: Vec<u8>) -> Result<Self, Error> {
265 Self::try_parse_storage(
266 path,
267 PackData::from_owned(raw),
268 CommitGraphParseMode::Verify,
269 )
270 }
271
272 fn try_parse_storage(
273 path: PathBuf,
274 storage: Arc<PackData>,
275 mode: CommitGraphParseMode,
276 ) -> Result<Self, Error> {
277 let raw = storage.as_ref();
278 if raw.len() < 28 {
279 return Err(Error::CorruptObject(
280 "commit-graph file too small".to_owned(),
281 ));
282 }
283 let hash_len = match commit_graph_hash_len(raw[5]) {
286 Some(n) => n,
287 None => {
288 return Err(Error::CorruptObject(format!(
289 "commit-graph version/hash not supported (version {} hash {})",
290 raw[4], raw[5]
291 )))
292 }
293 };
294 let algo = HashAlgo::try_from(raw[5]).map_err(|_| {
295 Error::CorruptObject(format!(
296 "commit-graph version/hash not supported (version {} hash {})",
297 raw[4], raw[5]
298 ))
299 })?;
300 if mode == CommitGraphParseMode::Verify {
301 verify_trailer(algo, raw)
302 .map_err(|_| Error::CorruptObject("commit-graph incorrect checksum".to_owned()))?;
303 }
304 let body_len = raw.len().saturating_sub(hash_len);
305 let body = &raw[..body_len];
306 if body.len() < 8 || &body[0..4] != SIGNATURE {
307 return Err(Error::CorruptObject(
308 "commit-graph has bad signature".to_owned(),
309 ));
310 }
311 if body[4] != GRAPH_VERSION {
312 return Err(Error::CorruptObject(format!(
313 "commit-graph version/hash not supported (version {} hash {})",
314 body[4], body[5]
315 )));
316 }
317 let num_chunks = body[6] as usize;
318 let toc_start: usize = 8;
319 let toc_end = num_chunks
320 .checked_add(1)
321 .and_then(|n| n.checked_mul(12))
322 .and_then(|n| toc_start.checked_add(n))
323 .ok_or_else(|| {
324 Error::CorruptObject("commit-graph truncated at chunk table".to_owned())
325 })?;
326 if body.len() < toc_end {
327 return Err(Error::CorruptObject(
328 "commit-graph truncated at chunk table".to_owned(),
329 ));
330 }
331
332 let mut fanout_off = None;
333 let mut oid_lookup_off = None;
334 let mut commit_data_off = None;
335 let mut generation_off = None;
336 let mut generation_overflow_off = None;
337 let mut bloom_idx_off = None;
338 let mut bloom_data_range = None;
339 let mut base_graphs_off = None;
340 let mut extra_edges_off = None;
341 let mut extra_edges_end = None;
342 let mut chunk_offsets: Vec<usize> = Vec::new();
343 let mut toc_entries: Vec<(u32, usize)> = Vec::with_capacity(num_chunks);
344
345 for i in 0..num_chunks {
346 let e = toc_start + i * 12;
347 let id = u32::from_be_bytes(
348 body[e..e + 4]
349 .try_into()
350 .map_err(|_| Error::CorruptObject("commit-graph bad TOC".to_owned()))?,
351 );
352 let off = parse_toc_offset(
353 body.len(),
354 body[e + 4..e + 12]
355 .try_into()
356 .map_err(|_| Error::CorruptObject("commit-graph bad TOC".to_owned()))?,
357 )?;
358 toc_entries.push((id, off));
359 chunk_offsets.push(off);
360 match id {
361 CHUNK_OID_FANOUT => fanout_off = Some(off),
362 CHUNK_OID_LOOKUP => oid_lookup_off = Some(off),
363 CHUNK_COMMIT_DATA => commit_data_off = Some(off),
364 CHUNK_GENERATION_DATA => generation_off = Some(off),
365 CHUNK_GENERATION_DATA_OVERFLOW => generation_overflow_off = Some(off),
366 CHUNK_BLOOM_INDEXES => bloom_idx_off = Some(off),
367 CHUNK_EXTRA_EDGES => {
368 extra_edges_off = Some(off);
369 extra_edges_end = Some(if i + 1 < num_chunks {
370 let e2 = toc_start
371 .checked_add((i + 1).checked_mul(12).ok_or_else(|| {
372 Error::CorruptObject("commit-graph bad TOC".to_owned())
373 })?)
374 .ok_or_else(|| {
375 Error::CorruptObject("commit-graph bad TOC".to_owned())
376 })?;
377 parse_toc_offset(
378 body.len(),
379 body[e2 + 4..e2 + 12].try_into().map_err(|_| {
380 Error::CorruptObject("commit-graph bad TOC".to_owned())
381 })?,
382 )?
383 } else {
384 let term = toc_start
385 .checked_add(num_chunks.checked_mul(12).ok_or_else(|| {
386 Error::CorruptObject("commit-graph bad TOC".to_owned())
387 })?)
388 .ok_or_else(|| {
389 Error::CorruptObject("commit-graph bad TOC".to_owned())
390 })?;
391 parse_toc_offset(
392 body.len(),
393 body[term + 4..term + 12].try_into().map_err(|_| {
394 Error::CorruptObject("commit-graph bad TOC".to_owned())
395 })?,
396 )?
397 });
398 }
399 CHUNK_BASE_GRAPHS => base_graphs_off = Some(off),
400 CHUNK_BLOOM_DATA => {
401 let end = if i + 1 < num_chunks {
402 let e2 = toc_start
403 .checked_add((i + 1).checked_mul(12).ok_or_else(|| {
404 Error::CorruptObject("commit-graph bad TOC".to_owned())
405 })?)
406 .ok_or_else(|| {
407 Error::CorruptObject("commit-graph bad TOC".to_owned())
408 })?;
409 parse_toc_offset(
410 body.len(),
411 body[e2 + 4..e2 + 12].try_into().map_err(|_| {
412 Error::CorruptObject("commit-graph bad TOC".to_owned())
413 })?,
414 )?
415 } else {
416 let term = toc_start
417 .checked_add(num_chunks.checked_mul(12).ok_or_else(|| {
418 Error::CorruptObject("commit-graph bad TOC".to_owned())
419 })?)
420 .ok_or_else(|| {
421 Error::CorruptObject("commit-graph bad TOC".to_owned())
422 })?;
423 parse_toc_offset(
424 body.len(),
425 body[term + 4..term + 12].try_into().map_err(|_| {
426 Error::CorruptObject("commit-graph bad TOC".to_owned())
427 })?,
428 )?
429 };
430 let len = end.checked_sub(off).ok_or_else(|| {
431 Error::CorruptObject("commit-graph chunk layout invalid".to_owned())
432 })?;
433 bloom_data_range = Some((off, len));
434 }
435 _ => {}
436 }
437 }
438 let file_end_pos = toc_start
439 .checked_add(
440 num_chunks
441 .checked_mul(12)
442 .ok_or_else(|| Error::CorruptObject("commit-graph bad file end".to_owned()))?,
443 )
444 .and_then(|t| t.checked_add(4))
445 .ok_or_else(|| Error::CorruptObject("commit-graph bad file end".to_owned()))?;
446 let file_end = parse_toc_offset(
447 body.len(),
448 body[file_end_pos..file_end_pos + 8]
449 .try_into()
450 .map_err(|_| Error::CorruptObject("commit-graph bad file end".to_owned()))?,
451 )?;
452 if file_end > body.len() {
453 return Err(commit_graph_offset_out_of_range());
454 }
455 chunk_offsets.push(file_end);
456 chunk_offsets.sort_unstable();
457 chunk_offsets.dedup();
458
459 fn chunk_byte_range(
460 start: usize,
461 toc_entries: &[(u32, usize)],
462 file_end: usize,
463 ) -> Result<usize, Error> {
464 let mut ends: Vec<usize> = toc_entries
465 .iter()
466 .map(|&(_, o)| o)
467 .filter(|&o| o > start)
468 .collect();
469 ends.sort_unstable();
470 let end = ends.first().copied().unwrap_or(file_end);
471 if end < start {
472 return Err(Error::CorruptObject(
473 "commit-graph chunk layout invalid".to_owned(),
474 ));
475 }
476 Ok(end)
477 }
478
479 if mode == CommitGraphParseMode::Verify {
480 if let Some(gda) = generation_off {
481 let gda_end = chunk_byte_range(gda, &toc_entries, file_end)?;
482 let gda_len = gda_end.saturating_sub(gda);
483 let fo = fanout_off.ok_or_else(|| {
484 Error::CorruptObject("commit-graph missing fanout".to_owned())
485 })?;
486 let num_commits_probe = fanout_num_commits(body, fo)?;
487 let expected = num_commits_probe as usize * 4;
488 if gda_len < expected {
489 return Err(Error::CorruptObject(
490 "commit-graph generation data chunk is too small".to_owned(),
491 ));
492 }
493 let gda_slice = body.get(gda..gda + expected).ok_or_else(|| {
494 Error::CorruptObject("commit-graph generation data OOB".to_owned())
495 })?;
496 let mut max_overflow_idx: Option<u32> = None;
497 for w in 0..num_commits_probe as usize {
498 let v = u32::from_be_bytes(gda_slice[w * 4..w * 4 + 4].try_into().map_err(
499 |_| Error::CorruptObject("commit-graph GDA2 corrupt".to_owned()),
500 )?);
501 if v & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW != 0 {
502 let pos = v ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW;
503 max_overflow_idx = Some(match max_overflow_idx {
504 None => pos,
505 Some(m) => m.max(pos),
506 });
507 }
508 }
509 if let Some(pos) = max_overflow_idx {
510 let Some(gdo_start) = generation_overflow_off else {
511 return Err(Error::CorruptObject(
512 "commit-graph requires overflow generation data but has none"
513 .to_owned(),
514 ));
515 };
516 let gdo_end = chunk_byte_range(gdo_start, &toc_entries, file_end)?;
517 let overflow_bytes = gdo_end.saturating_sub(gdo_start);
518 let n_slots = overflow_bytes / 8;
519 if n_slots <= pos as usize {
520 return Err(Error::CorruptObject(
521 "commit-graph overflow generation data is too small".to_owned(),
522 ));
523 }
524 }
525 }
526 }
527 let bidx_len = bloom_idx_off.and_then(|b| {
528 chunk_offsets
529 .iter()
530 .find(|&&o| o > b)
531 .map(|&next| next.saturating_sub(b))
532 });
533
534 let fanout_off = fanout_off.ok_or_else(|| {
535 Error::CorruptObject("commit-graph missing OID fanout chunk".to_owned())
536 })?;
537 let oid_lookup_off = oid_lookup_off.ok_or_else(|| {
538 Error::CorruptObject("commit-graph missing OID lookup chunk".to_owned())
539 })?;
540 let commit_data_off = commit_data_off.ok_or_else(|| {
541 Error::CorruptObject("commit-graph missing commit data chunk".to_owned())
542 })?;
543 checked_add_end(oid_lookup_off, 4, body.len())?;
544 let num_commits = fanout_num_commits(body, fanout_off)?;
545 let oid_lookup_bytes = (num_commits as usize)
546 .checked_mul(hash_len)
547 .ok_or_else(commit_graph_chunk_oob)?;
548 checked_add_end(oid_lookup_off, oid_lookup_bytes, body.len())?;
549 let graph_data_width = hash_len
550 .checked_add(16)
551 .ok_or_else(commit_graph_chunk_oob)?;
552 let cdat_bytes = (num_commits as usize)
553 .checked_mul(graph_data_width)
554 .ok_or_else(commit_graph_chunk_oob)?;
555 checked_add_end(commit_data_off, cdat_bytes, body.len())?;
556
557 if mode == CommitGraphParseMode::Verify {
558 for bucket in 0usize..255 {
559 let a_off = fanout_off
560 .checked_add(bucket.checked_mul(4).ok_or_else(commit_graph_chunk_oob)?)
561 .ok_or_else(commit_graph_chunk_oob)?;
562 let a =
563 u32::from_be_bytes(body_slice(body, a_off, 4)?.try_into().map_err(|_| {
564 Error::CorruptObject("commit-graph fanout corrupt".to_owned())
565 })?);
566 let b_off = fanout_off
567 .checked_add(
568 (bucket + 1)
569 .checked_mul(4)
570 .ok_or_else(commit_graph_chunk_oob)?,
571 )
572 .ok_or_else(commit_graph_chunk_oob)?;
573 let b =
574 u32::from_be_bytes(body_slice(body, b_off, 4)?.try_into().map_err(|_| {
575 Error::CorruptObject("commit-graph fanout corrupt".to_owned())
576 })?);
577 if a > b {
578 return Err(Error::CorruptObject(
579 "commit-graph fanout value non-monotonic".to_owned(),
580 ));
581 }
582 }
583
584 for lex in 1..num_commits {
585 let off_a = oid_lookup_off
586 .checked_add(
587 (lex as usize - 1)
588 .checked_mul(hash_len)
589 .ok_or_else(commit_graph_chunk_oob)?,
590 )
591 .ok_or_else(commit_graph_chunk_oob)?;
592 let off_b = oid_lookup_off
593 .checked_add(
594 (lex as usize)
595 .checked_mul(hash_len)
596 .ok_or_else(commit_graph_chunk_oob)?,
597 )
598 .ok_or_else(commit_graph_chunk_oob)?;
599 let slice_a = body_slice(body, off_a, hash_len)?;
600 let slice_b = body_slice(body, off_b, hash_len)?;
601 if slice_a >= slice_b {
602 return Err(Error::CorruptObject(
603 "commit-graph incorrect OID order".to_owned(),
604 ));
605 }
606 }
607
608 if base_graphs_off.is_none() {
609 validate_single_layer_parent_indices(
610 body,
611 num_commits,
612 commit_data_off,
613 graph_data_width,
614 hash_len,
615 extra_edges_off,
616 &toc_entries,
617 file_end,
618 )?;
619 }
620 }
621
622 let read_generation_data = generation_off.is_some();
623 let mut bloom_settings = None;
624 let mut chunk_bloom_data = None;
625 if let (Some(_bidx), Some((bdat_off, bdat_len))) = (bloom_idx_off, bloom_data_range) {
626 if bdat_len < BLOOM_HEADER {
627 } else if bdat_off + bdat_len <= body.len() {
628 let hdr = &body[bdat_off..bdat_off + BLOOM_HEADER];
629 let hash_version: [u8; 4] = hdr[0..4]
630 .try_into()
631 .map_err(|_| Error::CorruptObject("Bloom header corrupt".to_owned()))?;
632 let num_hashes: [u8; 4] = hdr[4..8]
633 .try_into()
634 .map_err(|_| Error::CorruptObject("Bloom header corrupt".to_owned()))?;
635 let bits_per_entry: [u8; 4] = hdr[8..12]
636 .try_into()
637 .map_err(|_| Error::CorruptObject("Bloom header corrupt".to_owned()))?;
638 bloom_settings = Some(BloomFilterSettings {
639 hash_version: u32::from_be_bytes(hash_version),
640 num_hashes: u32::from_be_bytes(num_hashes),
641 bits_per_entry: u32::from_be_bytes(bits_per_entry),
642 max_changed_paths: 512,
643 });
644 chunk_bloom_data = Some((bdat_off, bdat_len));
645 }
646 }
647
648 let bloom_indexes_ok = if let (Some(bidx), Some(bsize)) = (bloom_idx_off, bidx_len) {
649 !(bsize / 4 != num_commits as usize || bidx + bsize > body.len())
650 } else {
651 false
652 };
653 let bloom_pair_ok = bloom_settings.is_some()
654 && chunk_bloom_data.is_some()
655 && bloom_indexes_ok
656 && chunk_bloom_data.is_some_and(|(_, len)| len >= BLOOM_HEADER);
657
658 let (chunk_bloom_indexes, bloom_settings) = if bloom_pair_ok {
659 (bloom_idx_off, bloom_settings)
660 } else {
661 (None, None)
662 };
663 let chunk_bloom_data = if bloom_pair_ok {
664 chunk_bloom_data
665 } else {
666 None
667 };
668
669 let base_chunk_size = match base_graphs_off {
670 Some(off) => {
671 let end = chunk_byte_range(off, &toc_entries, file_end)?;
672 end.saturating_sub(off)
673 }
674 None => 0,
675 };
676
677 Ok(Self {
678 path,
679 file: storage,
680 body_len,
681 num_commits,
682 oid_lookup_off,
683 chunk_commit_data_off: commit_data_off,
684 chunk_generation_data: generation_off,
685 chunk_generation_overflow: generation_overflow_off,
686 read_generation_data,
687 chunk_bloom_indexes,
688 chunk_bloom_data,
689 bloom_settings,
690 bloom_disabled: false,
691 base_chunk_size,
692 hash_len,
693 algo,
694 extra_edges_off,
695 extra_edges_end,
696 })
697 }
698
699 fn parse(path: PathBuf, raw: Vec<u8>) -> Option<Self> {
700 Self::try_parse(path, raw).ok()
701 }
702
703 fn oid_at_lex(&self, lex_index: u32) -> Option<ObjectId> {
704 if lex_index >= self.num_commits {
705 return None;
706 }
707 let off = self.oid_lookup_off + lex_index as usize * self.hash_len;
708 ObjectId::from_bytes(self.body().get(off..off + self.hash_len)?).ok()
709 }
710
711 fn parent_global_positions(&self, lex_index: u32) -> Option<Vec<u32>> {
718 let (p1, p2, _) = self.cdat_record(lex_index)?;
719 let mut out = Vec::new();
720 if p1 == GRAPH_PARENT_NONE {
721 return Some(out);
722 }
723 if p1 & GRAPH_EXTRA_EDGES_NEEDED != 0 {
724 return None;
725 }
726 out.push(p1);
727 if p2 == GRAPH_PARENT_NONE {
728 return Some(out);
729 }
730 if p2 & GRAPH_EXTRA_EDGES_NEEDED != 0 {
731 let start = (p2 & !GRAPH_EXTRA_EDGES_NEEDED) as usize;
732 let edge_off = self.extra_edges_off?;
733 let edge_end = self.extra_edges_end?;
734 let edge_len = edge_end.saturating_sub(edge_off);
735 if start.saturating_mul(4) >= edge_len {
736 return None;
737 }
738 let mut idx = start;
739 loop {
740 if idx.saturating_mul(4).saturating_add(4) > edge_len {
741 return None;
742 }
743 let ev = u32::from_be_bytes(
744 self.body()[edge_off + idx * 4..edge_off + idx * 4 + 4]
745 .try_into()
746 .ok()?,
747 );
748 let parent = ev & !GRAPH_LAST_EDGE;
749 if parent != GRAPH_PARENT_NONE && parent & GRAPH_EXTRA_EDGES_NEEDED == 0 {
750 out.push(parent);
751 }
752 if ev & GRAPH_LAST_EDGE != 0 {
753 break;
754 }
755 idx += 1;
756 }
757 return Some(out);
758 }
759 out.push(p2);
760 Some(out)
761 }
762
763 fn cdat_root_tree(&self, lex_index: u32) -> Option<ObjectId> {
765 if lex_index >= self.num_commits {
766 return None;
767 }
768 let base = self.chunk_commit_data_off + lex_index as usize * (self.hash_len + 16);
769 ObjectId::from_bytes(self.body().get(base..base + self.hash_len)?).ok()
770 }
771
772 fn cdat_record(&self, lex_index: u32) -> Option<(u32, u32, i64)> {
773 if lex_index >= self.num_commits {
774 return None;
775 }
776 let base = self.chunk_commit_data_off + lex_index as usize * (self.hash_len + 16);
777 let p1_off = base + self.hash_len;
778 let p1 = u32::from_be_bytes(self.body().get(p1_off..p1_off + 4)?.try_into().ok()?);
779 let p2 = u32::from_be_bytes(self.body().get(p1_off + 4..p1_off + 8)?.try_into().ok()?);
780 let gen_time =
781 u64::from_be_bytes(self.body().get(p1_off + 8..p1_off + 16)?.try_into().ok()?);
782 let commit_time = (gen_time & 0x3_FFFF_FFFF) as i64;
783 Some((p1, p2, commit_time))
784 }
785
786 fn corrected_commit_generation(&self, lex_index: u32) -> Option<u64> {
788 if !self.read_generation_data {
789 return None;
790 }
791 let (_, _, commit_time) = self.cdat_record(lex_index)?;
792 let gda_off = self.chunk_generation_data?;
793 if lex_index >= self.num_commits {
794 return None;
795 }
796 let start = gda_off + lex_index as usize * 4;
797 let word = u32::from_be_bytes(self.body().get(start..start + 4)?.try_into().ok()?);
798 let offset = if word & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW != 0 {
799 let slot = (word ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) as usize;
800 let gdo_off = self.chunk_generation_overflow?;
801 let off = gdo_off + slot * 8;
802 let hi = u32::from_be_bytes(self.body().get(off..off + 4)?.try_into().ok()?);
803 let lo = u32::from_be_bytes(self.body().get(off + 4..off + 8)?.try_into().ok()?);
804 (u64::from(hi) << 32) | u64::from(lo)
805 } else {
806 u64::from(word)
807 };
808 Some((commit_time as u64).saturating_add(offset))
809 }
810
811 fn bsearch_oid(&self, oid: &ObjectId) -> Option<u32> {
812 let mut lo = 0u32;
813 let mut hi = self.num_commits;
814 let bytes = oid.as_bytes();
815 while lo < hi {
816 let mid = (lo + hi) / 2;
817 let off = self.oid_lookup_off + mid as usize * self.hash_len;
818 let slice = &self.body()[off..off + self.hash_len];
819 match slice.cmp(bytes) {
820 std::cmp::Ordering::Less => lo = mid + 1,
821 std::cmp::Ordering::Greater => hi = mid,
822 std::cmp::Ordering::Equal => return Some(mid),
823 }
824 }
825 None
826 }
827
828 fn disable_bloom(&mut self) {
829 self.chunk_bloom_indexes = None;
830 self.chunk_bloom_data = None;
831 self.bloom_settings = None;
832 self.bloom_disabled = true;
833 }
834
835 fn layer_display_id(&self) -> String {
836 self.path
837 .file_stem()
838 .and_then(|s| s.to_str())
839 .map(|s| s.strip_prefix("graph-").unwrap_or(s).to_string())
840 .unwrap_or_else(|| "commit-graph".to_string())
841 }
842
843 fn bloom_filter_slice(&self, lex_index: u32) -> Option<&[u8]> {
844 let _settings = self.bloom_settings.as_ref()?;
845 let bidx_base = self.chunk_bloom_indexes?;
846 let (bdat_off, bdat_total) = self.chunk_bloom_data?;
847 let _graph_warn = warn_path_for_graph_file(self.path.as_path());
848 if lex_index >= self.num_commits {
849 return None;
850 }
851 let payload_len = bdat_total.saturating_sub(BLOOM_HEADER);
852 let end_rel = u32::from_be_bytes(
853 self.body()[bidx_base + lex_index as usize * 4..bidx_base + lex_index as usize * 4 + 4]
854 .try_into()
855 .ok()?,
856 ) as usize;
857 let start_rel = if lex_index == 0 {
858 0usize
859 } else {
860 u32::from_be_bytes(
861 self.body()[bidx_base + (lex_index as usize - 1) * 4
862 ..bidx_base + (lex_index as usize - 1) * 4 + 4]
863 .try_into()
864 .ok()?,
865 ) as usize
866 };
867 let max_payload = payload_len;
873 if end_rel > max_payload {
874 return None;
875 }
876 if start_rel > max_payload {
877 return None;
878 }
879 if end_rel < start_rel {
880 return None;
881 }
882 let data_base = bdat_off + BLOOM_HEADER;
883 let abs_start = data_base + start_rel;
884 let abs_end = data_base + end_rel;
885 if abs_end > bdat_off + bdat_total || abs_start > abs_end {
886 return None;
887 }
888 Some(&self.body()[abs_start..abs_end])
889 }
890}
891
892#[derive(Debug, Clone, Copy, PartialEq, Eq)]
894pub enum BloomPrecheck {
895 Inapplicable,
897 NotInGraph,
899 FilterNotPresent,
901 DefinitelyNot,
903 Maybe,
905}
906
907#[derive(Debug, Default, Clone)]
909pub struct BloomWalkStats {
910 pub filter_not_present: u32,
912 pub maybe: u32,
914 pub definitely_not: u32,
916 pub false_positive: u32,
918}
919
920impl BloomWalkStats {
921 pub fn record_precheck(&mut self, pre: BloomPrecheck) {
923 match pre {
924 BloomPrecheck::Inapplicable | BloomPrecheck::NotInGraph => {}
925 BloomPrecheck::FilterNotPresent => self.filter_not_present += 1,
926 BloomPrecheck::DefinitelyNot => self.definitely_not += 1,
927 BloomPrecheck::Maybe => self.maybe += 1,
928 }
929 }
930
931 pub fn record_false_positive(&mut self) {
933 self.false_positive += 1;
934 }
935}
936
937pub type BloomWalkStatsHandle = Arc<Mutex<BloomWalkStats>>;
939
940pub(crate) const UNRANKED_GENERATION: u64 = (1u64 << 63) - 1;
943
944#[derive(Debug, Clone)]
946pub(crate) struct GraphWalkEntry {
947 pub(crate) tree: ObjectId,
949 pub(crate) parents: Vec<ObjectId>,
951 pub(crate) time: i64,
953 pub(crate) generation: u64,
955}
956
957#[derive(Debug, Clone)]
959pub struct CommitGraphChain {
960 layers: Vec<CommitGraphLayer>,
961}
962
963#[doc(hidden)]
965pub fn commit_graph_chain_from_layers_for_test(layers: Vec<CommitGraphLayer>) -> CommitGraphChain {
966 CommitGraphChain { layers }
967}
968
969impl CommitGraphChain {
970 pub fn from_bytes(bytes: Vec<u8>) -> Result<Self, Error> {
988 if bytes.get(7).is_some_and(|&base_graphs| base_graphs != 0) {
989 return Err(Error::CorruptObject(
990 "commit-graph layer depends on base graphs".to_owned(),
991 ));
992 }
993 let layer = CommitGraphLayer::try_parse(PathBuf::from("commit-graph"), bytes)?;
994 if layer.base_chunk_size != 0 {
995 return Err(Error::CorruptObject(
996 "commit-graph layer depends on base graphs".to_owned(),
997 ));
998 }
999 Ok(Self {
1000 layers: vec![layer],
1001 })
1002 }
1003
1004 #[must_use]
1006 pub fn hash_algo(&self) -> HashAlgo {
1007 self.layers
1008 .first()
1009 .map_or(HashAlgo::Sha1, |layer| layer.algo)
1010 }
1011
1012 #[must_use]
1014 pub fn top_layer_bloom_settings(&self) -> Option<BloomFilterSettings> {
1015 self.layers.first()?.bloom_settings
1016 }
1017
1018 #[must_use]
1020 pub fn total_commits(&self) -> u32 {
1021 self.layers.iter().map(|l| l.num_commits).sum()
1022 }
1023
1024 #[must_use]
1026 pub fn layer_paths_oldest_first(&self) -> Vec<PathBuf> {
1027 self.layers.iter().rev().map(|l| l.path.clone()).collect()
1028 }
1029
1030 #[must_use]
1032 pub fn num_layers(&self) -> usize {
1033 self.layers.len()
1034 }
1035
1036 #[must_use]
1038 pub fn layer_commit_counts_tip_first(&self) -> Vec<u32> {
1039 self.layers.iter().map(|l| l.num_commits).collect()
1040 }
1041
1042 #[must_use]
1044 pub fn layer_has_generation_data_tip_first(&self) -> Vec<bool> {
1045 self.layers
1046 .iter()
1047 .map(|l| l.chunk_generation_data.is_some())
1048 .collect()
1049 }
1050
1051 #[must_use]
1053 pub fn layer_hashes_tip_first(&self) -> Vec<String> {
1054 self.layers.iter().map(|l| l.layer_display_id()).collect()
1055 }
1056
1057 #[must_use]
1061 pub fn layer_object_dirs_tip_first(&self) -> Vec<PathBuf> {
1062 self.layers
1063 .iter()
1064 .map(|l| {
1065 let p = l.path.as_path();
1068 let info = if p
1069 .parent()
1070 .and_then(|d| d.file_name())
1071 .map(|n| n == "commit-graphs")
1072 .unwrap_or(false)
1073 {
1074 p.parent().and_then(|d| d.parent())
1075 } else {
1076 p.parent()
1077 };
1078 info.and_then(|d| d.parent())
1079 .map(Path::to_path_buf)
1080 .unwrap_or_else(|| PathBuf::from("."))
1081 })
1082 .collect()
1083 }
1084
1085 #[must_use]
1089 pub fn sub_chain_tip_first(&self, start: usize, end: usize) -> Option<Self> {
1090 let end = end.min(self.layers.len());
1091 if start >= end {
1092 return None;
1093 }
1094 Some(Self {
1095 layers: self.layers[start..end].to_vec(),
1096 })
1097 }
1098
1099 #[must_use]
1101 pub fn layer_oids(&self, tip_first_idx: usize) -> Vec<ObjectId> {
1102 let Some(layer) = self.layers.get(tip_first_idx) else {
1103 return Vec::new();
1104 };
1105 let mut out = Vec::with_capacity(layer.num_commits as usize);
1106 for i in 0..layer.num_commits {
1107 if let Some(oid) = layer.oid_at_lex(i) {
1108 out.push(oid);
1109 }
1110 }
1111 out
1112 }
1113
1114 pub fn try_load(objects_dir: &Path) -> Result<Option<Self>, Error> {
1119 Self::try_load_with_caches(objects_dir, None)
1120 }
1121
1122 pub fn try_load_with_caches(
1124 objects_dir: &Path,
1125 caches: Option<&crate::repo_caches::RepoCaches>,
1126 ) -> Result<Option<Self>, Error> {
1127 let info = objects_dir.join("info");
1128 let chain_path = info.join("commit-graphs").join("commit-graph-chain");
1129 if chain_path.is_file() {
1130 let algo =
1131 hash_algo_for_objects_dir(&crate::environment::Environment::empty(), objects_dir);
1132 let content = std::fs::read_to_string(&chain_path).map_err(Error::from)?;
1133 let mut layers = Vec::new();
1134 for line in content.lines() {
1135 let h = line.trim();
1136 if !commit_graph_chain_hash_is_valid(h, algo) {
1137 continue;
1138 }
1139 let graph_path = info.join("commit-graphs").join(format!("graph-{h}.graph"));
1140 let layer = CommitGraphLayer::try_open(graph_path)?;
1141 let n = layers.len();
1148 if n > 0 && layer.base_chunk_size / layer.hash_len < n {
1149 warn_once_for_base_chunk_too_small(caches, &layer.layer_display_id());
1150 break;
1151 }
1152 layers.push(layer);
1153 }
1154 if layers.is_empty() {
1155 return Ok(None);
1156 }
1157 layers.reverse();
1161 let mut chain = Self { layers };
1162 chain.validate_bloom_compatibility(caches);
1163 return Ok(Some(chain));
1164 }
1165 let single = info.join("commit-graph");
1166 if single.is_file() {
1167 let layer = CommitGraphLayer::try_open(single.clone())?;
1168 let mut chain = Self {
1169 layers: vec![layer],
1170 };
1171 chain.validate_bloom_compatibility(caches);
1172 return Ok(Some(chain));
1173 }
1174 Ok(None)
1175 }
1176
1177 pub fn load(objects_dir: &Path) -> Option<Self> {
1179 Self::try_load(objects_dir).ok().flatten()
1180 }
1181
1182 pub fn verify_on_disk(objects_dir: &Path) -> Result<(), Error> {
1190 let Some(chain) = Self::try_load(objects_dir)? else {
1191 return Ok(());
1192 };
1193 for path in chain.layer_paths_oldest_first() {
1194 let raw = std::fs::read(&path).map_err(Error::from)?;
1195 CommitGraphLayer::try_parse(path, raw)?;
1196 }
1197 Ok(())
1198 }
1199
1200 pub fn try_load_across(
1224 objects_dir: &Path,
1225 alt_dirs: &[PathBuf],
1226 ) -> Result<Option<Self>, Error> {
1227 Self::try_load_across_with_caches(objects_dir, alt_dirs, None)
1228 }
1229
1230 pub fn try_load_across_with_caches(
1236 objects_dir: &Path,
1237 alt_dirs: &[PathBuf],
1238 caches: Option<&crate::repo_caches::RepoCaches>,
1239 ) -> Result<Option<Self>, Error> {
1240 let layer_dir = |dir: &Path| dir.join("info").join("commit-graphs");
1241 let resolve_layer = |hash: &str| -> Option<PathBuf> {
1242 let name = format!("graph-{hash}.graph");
1243 let local = layer_dir(objects_dir).join(&name);
1244 if local.is_file() {
1245 return Some(local);
1246 }
1247 alt_dirs
1248 .iter()
1249 .map(|d| layer_dir(d).join(&name))
1250 .find(|p| p.is_file())
1251 };
1252
1253 let local_chain = layer_dir(objects_dir).join("commit-graph-chain");
1261 let local_single = objects_dir.join("info").join("commit-graph");
1262 let chain_path = if local_chain.is_file() {
1263 local_chain
1264 } else if local_single.is_file() {
1265 return Self::try_load_with_caches(objects_dir, caches);
1266 } else {
1267 match alt_dirs
1268 .iter()
1269 .map(PathBuf::as_path)
1270 .find(|d| layer_dir(d).join("commit-graph-chain").is_file())
1271 {
1272 Some(owner) => layer_dir(owner).join("commit-graph-chain"),
1273 None => return Ok(None),
1274 }
1275 };
1276
1277 let algo =
1278 hash_algo_for_objects_dir(&crate::environment::Environment::empty(), objects_dir);
1279 let content = std::fs::read_to_string(&chain_path).map_err(Error::from)?;
1280 let mut layers = Vec::new();
1281 for line in content.lines() {
1282 let h = line.trim();
1283 if !commit_graph_chain_hash_is_valid(h, algo) {
1284 continue;
1285 }
1286 let graph_path = resolve_layer(h).ok_or_else(|| {
1287 Error::from(std::io::Error::new(
1288 std::io::ErrorKind::NotFound,
1289 format!("commit-graph layer graph-{h}.graph not found"),
1290 ))
1291 })?;
1292 let layer = CommitGraphLayer::try_open(graph_path)?;
1293 let n = layers.len();
1294 if n > 0 && layer.base_chunk_size / layer.hash_len < n {
1295 warn_once_for_base_chunk_too_small(caches, &layer.layer_display_id());
1296 break;
1297 }
1298 layers.push(layer);
1299 }
1300 if layers.is_empty() {
1301 return Ok(None);
1302 }
1303 layers.reverse();
1304 let mut chain = Self { layers };
1305 chain.validate_bloom_compatibility(caches);
1306 Ok(Some(chain))
1307 }
1308
1309 fn validate_bloom_compatibility(&mut self, caches: Option<&crate::repo_caches::RepoCaches>) {
1310 let mut ref_settings: Option<BloomFilterSettings> = None;
1316 for layer in &mut self.layers {
1317 let Some(bs) = layer.bloom_settings else {
1318 continue;
1319 };
1320 match ref_settings {
1321 None => ref_settings = Some(bs),
1322 Some(r) => {
1323 if r.hash_version != bs.hash_version
1324 || r.num_hashes != bs.num_hashes
1325 || r.bits_per_entry != bs.bits_per_entry
1326 {
1327 let id = layer.layer_display_id();
1328 warn_once_for_disabled_bloom_layer(caches, &id);
1334 layer.disable_bloom();
1335 }
1336 }
1337 }
1338 }
1339 }
1340
1341 pub fn existing_filter_bytes(
1347 &self,
1348 oid: &ObjectId,
1349 want: &BloomFilterSettings,
1350 ) -> Option<Vec<u8>> {
1351 let (layer_idx, lex) = self.find_commit(oid)?;
1352 let layer = &self.layers[layer_idx];
1353 let settings = layer.bloom_settings.as_ref()?;
1354 if settings.hash_version != want.hash_version
1355 || settings.num_hashes != want.num_hashes
1356 || settings.bits_per_entry != want.bits_per_entry
1357 {
1358 return None;
1359 }
1360 match layer.bloom_filter_slice(lex) {
1367 Some(s) if !s.is_empty() => Some(s.to_vec()),
1368 _ => None,
1369 }
1370 }
1371
1372 pub fn upgradable_filter_bytes(
1378 &self,
1379 oid: &ObjectId,
1380 want: &BloomFilterSettings,
1381 ) -> Option<Vec<u8>> {
1382 let (layer_idx, lex) = self.find_commit(oid)?;
1383 let layer = &self.layers[layer_idx];
1384 let settings = layer.bloom_settings.as_ref()?;
1385 if settings.hash_version == want.hash_version {
1386 return None;
1387 }
1388 if settings.num_hashes != want.num_hashes || settings.bits_per_entry != want.bits_per_entry
1389 {
1390 return None;
1391 }
1392 match layer.bloom_filter_slice(lex) {
1393 Some(s) if !s.is_empty() => Some(s.to_vec()),
1394 _ => None,
1395 }
1396 }
1397
1398 pub(crate) fn bloom_settings_for_commit(&self, oid: &ObjectId) -> Option<BloomFilterSettings> {
1400 let (layer_idx, _) = self.find_commit(oid)?;
1401 self.layers[layer_idx].bloom_settings
1402 }
1403
1404 pub fn find_commit(&self, oid: &ObjectId) -> Option<(usize, u32)> {
1406 for (i, layer) in self.layers.iter().enumerate() {
1407 if let Some(lex) = layer.bsearch_oid(oid) {
1408 return Some((i, lex));
1409 }
1410 }
1411 None
1412 }
1413
1414 pub fn global_position(&self, oid: &ObjectId) -> Option<u32> {
1416 let (layer_idx, lex) = self.find_commit(oid)?;
1417 let below: u32 = self.layers[layer_idx + 1..]
1418 .iter()
1419 .map(|l| l.num_commits)
1420 .sum();
1421 Some(below + lex)
1422 }
1423
1424 fn oid_at_global_position(&self, pos: u32) -> Option<ObjectId> {
1427 let mut remaining = pos;
1429 for layer in self.layers.iter().rev() {
1430 if remaining < layer.num_commits {
1431 return layer.oid_at_lex(remaining);
1432 }
1433 remaining -= layer.num_commits;
1434 }
1435 None
1436 }
1437
1438 pub fn graph_commit(&self, oid: &ObjectId) -> Option<(Vec<ObjectId>, i64)> {
1441 let (layer_idx, lex) = self.find_commit(oid)?;
1442 self.graph_commit_at(layer_idx, lex)
1443 }
1444
1445 fn graph_commit_at(&self, layer_idx: usize, lex: u32) -> Option<(Vec<ObjectId>, i64)> {
1446 let layer = &self.layers[layer_idx];
1447 let (_, _, commit_time) = layer.cdat_record(lex)?;
1448 let oids = layer
1449 .parent_global_positions(lex)?
1450 .into_iter()
1451 .filter_map(|pos| self.oid_at_global_position(pos))
1452 .collect();
1453 Some((oids, commit_time))
1454 }
1455
1456 pub(crate) fn graph_commit_tree(&self, oid: &ObjectId) -> Option<ObjectId> {
1458 let (layer_idx, lex) = self.find_commit(oid)?;
1459 self.layers[layer_idx].cdat_root_tree(lex)
1460 }
1461
1462 pub(crate) fn graph_commit_meta(&self, oid: &ObjectId) -> Option<(ObjectId, Vec<ObjectId>)> {
1464 let (layer_idx, lex) = self.find_commit(oid)?;
1465 let tree = self.layers[layer_idx].cdat_root_tree(lex)?;
1466 let (parents, _) = self.graph_commit_at(layer_idx, lex)?;
1467 Some((tree, parents))
1468 }
1469
1470 pub(crate) fn graph_walk_entry(&self, oid: &ObjectId) -> Option<GraphWalkEntry> {
1473 let (layer_idx, lex) = self.find_commit(oid)?;
1474 let layer = &self.layers[layer_idx];
1475 let tree = layer.cdat_root_tree(lex)?;
1476 let (parents, time) = self.graph_commit_at(layer_idx, lex)?;
1477 Some(GraphWalkEntry {
1478 tree,
1479 parents,
1480 time,
1481 generation: layer
1482 .corrected_commit_generation(lex)
1483 .unwrap_or(UNRANKED_GENERATION),
1484 })
1485 }
1486
1487 #[must_use]
1489 pub fn commit_generation_for_walk(&self, oid: &ObjectId) -> u64 {
1490 let Some((layer_idx, lex)) = self.find_commit(oid) else {
1491 return UNRANKED_GENERATION;
1492 };
1493 self.layers[layer_idx]
1494 .corrected_commit_generation(lex)
1495 .unwrap_or(UNRANKED_GENERATION)
1496 }
1497
1498 #[must_use]
1501 pub fn parent_global_positions(&self, oid: &ObjectId) -> Option<Vec<u32>> {
1502 let (layer_idx, lex) = self.find_commit(oid)?;
1503 self.layers[layer_idx].parent_global_positions(lex)
1504 }
1505
1506 #[must_use]
1510 pub fn count_reachable_commits(
1511 &self,
1512 tips: &[ObjectId],
1513 hide: &[ObjectId],
1514 first_parent_only: bool,
1515 shallow: &std::collections::HashSet<ObjectId>,
1516 ) -> Option<usize> {
1517 if self.total_commits() == 0 {
1518 return None;
1519 }
1520 let mut seen = self.reachable_positions(tips, first_parent_only, shallow)?;
1521 if hide.is_empty() {
1522 return Some(seen.len());
1523 }
1524 let hidden = self.reachable_positions(hide, first_parent_only, shallow)?;
1525 seen.retain(|pos| !hidden.contains(pos));
1526 Some(seen.len())
1527 }
1528
1529 fn reachable_positions(
1530 &self,
1531 tips: &[ObjectId],
1532 first_parent_only: bool,
1533 shallow: &std::collections::HashSet<ObjectId>,
1534 ) -> Option<std::collections::HashSet<u32>> {
1535 let total = self.total_commits();
1536 let mut stack: Vec<u32> = Vec::new();
1537 for tip in tips {
1538 stack.push(self.global_position(tip)?);
1539 }
1540 let mut seen = std::collections::HashSet::new();
1541 while let Some(pos) = stack.pop() {
1542 if pos >= total || !seen.insert(pos) {
1543 continue;
1544 }
1545 #[cfg(test)]
1546 crate::merge_base::intersect_walk_test_hooks::record_commit_parent_lookup();
1547 let oid = self.oid_at_global_position(pos)?;
1548 let parents = if shallow.contains(&oid) {
1549 Vec::new()
1550 } else {
1551 self.parent_global_positions(&oid)?
1552 };
1553 let parents = if first_parent_only {
1554 parents.into_iter().take(1).collect::<Vec<_>>()
1555 } else {
1556 parents
1557 };
1558 for p in parents {
1559 if p >= total {
1560 return None;
1561 }
1562 stack.push(p);
1563 }
1564 }
1565 Some(seen)
1566 }
1567
1568 pub fn all_oids_in_order(&self) -> Vec<ObjectId> {
1570 let mut out = Vec::new();
1571 for layer in self.layers.iter().rev() {
1572 for i in 0..layer.num_commits {
1573 if let Some(oid) = layer.oid_at_lex(i) {
1574 out.push(oid);
1575 }
1576 }
1577 }
1578 out
1579 }
1580
1581 pub fn bloom_precheck_for_paths(
1583 &self,
1584 _odb: &Odb,
1585 oid: ObjectId,
1586 pathspecs: &[String],
1587 bloom_cwd: Option<&str>,
1588 requested_hash_version: i32,
1589 read_changed_paths: bool,
1590 ) -> std::result::Result<BloomPrecheck, crate::error::Error> {
1591 if !read_changed_paths {
1592 return Ok(BloomPrecheck::Inapplicable);
1593 }
1594 let Some((layer_idx, lex)) = self.find_commit(&oid) else {
1595 return Ok(BloomPrecheck::NotInGraph);
1596 };
1597 let layer = &self.layers[layer_idx];
1598 let Some(settings) = layer.bloom_settings.as_ref() else {
1599 return Ok(BloomPrecheck::FilterNotPresent);
1600 };
1601 let effective_version = if requested_hash_version < 0 {
1602 settings.hash_version as i32
1603 } else {
1604 requested_hash_version
1605 };
1606 if effective_version != settings.hash_version as i32 {
1607 return Ok(BloomPrecheck::FilterNotPresent);
1608 }
1609
1610 let filter = match layer.bloom_filter_slice(lex) {
1616 Some(s) => s,
1617 None => return Ok(BloomPrecheck::FilterNotPresent),
1618 };
1619 if filter.is_empty() {
1620 return Ok(BloomPrecheck::FilterNotPresent);
1621 }
1622
1623 let mut any_pathspec_maybe = false;
1626 let mut checked_any_keys = false;
1627 for spec in pathspecs {
1628 if spec.is_empty() || crate::pathspec::pathspec_is_exclude(spec) {
1629 continue;
1630 }
1631 let Some(norm) = crate::pathspec::bloom_lookup_prefix_with_cwd(spec, bloom_cwd) else {
1632 continue;
1633 };
1634 let keys = bloom_keyvec_for_path(norm.as_str(), settings);
1635 if keys.is_empty() {
1636 continue;
1637 }
1638 checked_any_keys = true;
1639 let mut all_keys_maybe = true;
1640 for key in &keys {
1641 match bloom_filter_contains(key, filter, settings) {
1642 Ok(true) => {}
1643 Ok(false) => {
1644 all_keys_maybe = false;
1645 break;
1646 }
1647 Err(crate::bloom::BloomFilterInvalid) => {
1648 all_keys_maybe = true;
1649 break;
1650 }
1651 }
1652 }
1653 if all_keys_maybe {
1654 any_pathspec_maybe = true;
1655 break;
1656 }
1657 }
1658 if checked_any_keys && !any_pathspec_maybe {
1659 return Ok(BloomPrecheck::DefinitelyNot);
1660 }
1661 Ok(BloomPrecheck::Maybe)
1662 }
1663
1664 pub(crate) fn bloom_definitely_not_for_keys(
1666 &self,
1667 oid: ObjectId,
1668 path_keys: &[Vec<u32>],
1669 ) -> bool {
1670 if path_keys.is_empty() {
1671 return false;
1672 }
1673 let Some((layer_idx, lex)) = self.find_commit(&oid) else {
1674 return false;
1675 };
1676 let layer = &self.layers[layer_idx];
1677 let Some(settings) = layer.bloom_settings.as_ref() else {
1678 return false;
1679 };
1680 let Some(filter) = layer.bloom_filter_slice(lex) else {
1681 return false;
1682 };
1683 if filter.is_empty() {
1684 return false;
1685 }
1686 bloom_definitely_not_from_filter(filter, settings, path_keys)
1687 }
1688}
1689
1690pub(crate) fn bloom_definitely_not_from_filter(
1692 filter: &[u8],
1693 settings: &crate::bloom::BloomFilterSettings,
1694 path_keys: &[Vec<u32>],
1695) -> bool {
1696 use crate::bloom::bloom_filter_contains;
1697
1698 for key in path_keys {
1699 match bloom_filter_contains(key, filter, settings) {
1700 Ok(false) => return true,
1701 Ok(true) => {}
1702 Err(crate::bloom::BloomFilterInvalid) => return false,
1703 }
1704 }
1705 false
1706}
1707
1708pub fn diff_changed_paths_for_bloom(
1710 odb: &Odb,
1711 parent_tree: Option<ObjectId>,
1712 commit_tree: ObjectId,
1713) -> crate::error::Result<(Vec<String>, usize)> {
1714 use crate::diff::diff_trees;
1715 let entries = diff_trees(odb, parent_tree.as_ref(), Some(&commit_tree), "")?;
1716 let raw_len = entries.len();
1717 let mut paths = Vec::new();
1718 for e in entries {
1719 let p = e.path().to_string();
1720 if !p.is_empty() {
1721 paths.push(p);
1722 }
1723 }
1724 Ok((paths, raw_len))
1725}
1726
1727pub use crate::bloom::collect_changed_paths_for_bloom;
1729
1730pub fn bloom_filter_for_commit_write(
1732 odb: &Odb,
1733 parents: &[ObjectId],
1734 tree_oid: ObjectId,
1735 settings: &BloomFilterSettings,
1736) -> crate::error::Result<(Vec<u8>, BloomBuildOutcome)> {
1737 let (changed_paths_vec, raw_count) = if let Some(first_parent) = parents.first() {
1741 let p = odb.commit_tree(first_parent)?;
1742 diff_changed_paths_for_bloom(odb, Some(p), tree_oid)?
1743 } else {
1744 diff_changed_paths_for_bloom(odb, None, tree_oid)?
1745 };
1746 let set = collect_changed_paths_for_bloom(&changed_paths_vec);
1747 Ok(crate::bloom::build_bloom_filter_data(
1748 &set, raw_count, settings,
1749 ))
1750}
1751
1752fn tree_has_high_bit_paths(odb: &Odb, tree_oid: ObjectId) -> bool {
1757 let Ok(obj) = odb.read(&tree_oid) else {
1758 return true;
1760 };
1761 let Ok(entries) = crate::objects::parse_tree(&obj.data) else {
1762 return true;
1763 };
1764 for e in &entries {
1765 if e.name.iter().any(|&b| b & 0x80 != 0) {
1766 return true;
1767 }
1768 if e.mode == 0o040000 && tree_has_high_bit_paths(odb, e.oid) {
1769 return true;
1770 }
1771 }
1772 false
1773}
1774
1775pub fn commit_tree_has_high_bit_paths(odb: &Odb, commit_oid: ObjectId) -> bool {
1778 match odb.commit_tree(&commit_oid) {
1779 Ok(tree) => tree_has_high_bit_paths(odb, tree),
1780 Err(_) => true,
1781 }
1782}
1783
1784pub fn parse_graph_file(path: &Path) -> Option<ParsedGraphDump> {
1786 let raw = std::fs::read(path).ok()?;
1787 if raw.len() < 28 {
1788 return None;
1789 }
1790 let hash_len = commit_graph_hash_len(raw[5])?;
1791 let body = &raw[..raw.len() - hash_len];
1792 if body.len() < 8 || &body[0..4] != SIGNATURE {
1793 return None;
1794 }
1795 let header_word = u32::from_be_bytes(body[0..4].try_into().ok()?);
1796 let num_chunks = body[6] as usize;
1797 let toc_start = 8;
1798 let mut present: std::collections::HashSet<u32> = std::collections::HashSet::new();
1799 for i in 0..num_chunks {
1800 let e = toc_start + i * 12;
1801 let id = u32::from_be_bytes(body[e..e + 4].try_into().ok()?);
1802 present.insert(id);
1803 }
1804 let mut chunk_names: Vec<String> = Vec::new();
1807 for (id, label) in [
1808 (CHUNK_OID_FANOUT, "oid_fanout"),
1809 (CHUNK_OID_LOOKUP, "oid_lookup"),
1810 (CHUNK_COMMIT_DATA, "commit_metadata"),
1811 (CHUNK_GENERATION_DATA, "generation_data"),
1812 (CHUNK_GENERATION_DATA_OVERFLOW, "generation_data_overflow"),
1813 (CHUNK_EXTRA_EDGES, "extra_edges"),
1814 (CHUNK_BLOOM_INDEXES, "bloom_indexes"),
1815 (CHUNK_BLOOM_DATA, "bloom_data"),
1816 ] {
1817 if present.contains(&id) {
1818 chunk_names.push(label.to_string());
1819 }
1820 }
1821 let layer = CommitGraphLayer::parse(path.to_path_buf(), raw.clone())?;
1822 let bloom_opt = layer.bloom_settings.map(|s| {
1823 format!(
1824 " bloom({},{},{})",
1825 s.hash_version, s.bits_per_entry, s.num_hashes
1826 )
1827 });
1828 let mut options = String::new();
1829 if let Some(b) = bloom_opt {
1830 options.push_str(&b);
1831 }
1832 if layer.read_generation_data {
1833 options.push_str(" read_generation_data");
1834 }
1835 Some(ParsedGraphDump {
1836 header_word,
1837 version: body[4],
1838 hash_ver: body[5],
1839 num_chunks: body[6],
1840 reserved: body[7],
1841 num_commits: layer.num_commits,
1842 chunks: chunk_names.join(" "),
1843 options,
1844 })
1845}
1846
1847pub struct ParsedGraphDump {
1849 pub header_word: u32,
1851 pub version: u8,
1853 pub hash_ver: u8,
1855 pub num_chunks: u8,
1857 pub reserved: u8,
1859 pub num_commits: u32,
1861 pub chunks: String,
1863 pub options: String,
1865}
1866
1867pub fn dump_bloom_filters(path: &Path) -> Option<Vec<String>> {
1869 let raw = std::fs::read(path).ok()?;
1870 let layer = CommitGraphLayer::parse(path.to_path_buf(), raw)?;
1871 let mut out = Vec::new();
1872 for i in 0..layer.num_commits {
1873 let slice = layer.bloom_filter_slice(i).unwrap_or(&[]);
1874 if slice.is_empty() {
1875 out.push(String::new());
1876 } else {
1877 let hex: String = slice.iter().map(|b| format!("{b:02x}")).collect();
1878 out.push(hex);
1879 }
1880 }
1881 Some(out)
1882}
1883
1884#[cfg(test)]
1885mod tests {
1886 use std::collections::HashMap;
1887
1888 use crate::bloom::BloomFilterSettings;
1889 use crate::commit_graph_write::{build_commit_graph_bytes, load_commit_graph_commit_info};
1890 use crate::index::{Index, IndexEntry, MODE_REGULAR};
1891 use crate::objects::{serialize_commit, CommitData, ObjectId, ObjectKind};
1892 use crate::repo::{init_repository, Repository};
1893 use crate::write_tree::write_tree_from_index;
1894
1895 use crate::commit_graph_write::CommitGraphCommitInfo;
1896 use crate::objects::OidHashMap;
1897
1898 fn infos_std(
1899 infos: &OidHashMap<CommitGraphCommitInfo>,
1900 ) -> HashMap<ObjectId, CommitGraphCommitInfo> {
1901 infos.iter().map(|(k, v)| (*k, v.clone())).collect()
1902 }
1903
1904 fn empty_byte_filters() -> HashMap<ObjectId, Vec<u8>> {
1905 HashMap::new()
1906 }
1907
1908 fn one_commit_repo() -> (tempfile::TempDir, Repository, ObjectId) {
1909 let dir = tempfile::tempdir().expect("tempdir");
1910 let repo = init_repository(
1911 dir.path(),
1912 false,
1913 "main",
1914 None,
1915 crate::RefStorageFormat::Files,
1916 )
1917 .expect("init");
1918 let mut index = Index::new();
1919 index.hash_algo = repo.odb().hash_algo();
1920 let blob = repo.odb().write(ObjectKind::Blob, b"x\n").expect("blob");
1921 index.add_or_replace(IndexEntry {
1922 ctime_sec: 0,
1923 ctime_nsec: 0,
1924 mtime_sec: 0,
1925 mtime_nsec: 0,
1926 dev: 0,
1927 ino: 0,
1928 mode: MODE_REGULAR,
1929 uid: 0,
1930 gid: 0,
1931 size: 2,
1932 oid: blob,
1933 flags: 4,
1934 flags_extended: None,
1935 path: b"f.txt".to_vec(),
1936 base_index_pos: 0,
1937 });
1938 let tree = write_tree_from_index(&repo.odb(), &index, "").expect("tree");
1939 let raw = serialize_commit(&CommitData {
1940 tree,
1941 parents: vec![],
1942 author: "T <t@example.com> 100 +0000".into(),
1943 committer: "T <t@example.com> 100 +0000".into(),
1944 author_raw: Vec::new(),
1945 committer_raw: Vec::new(),
1946 encoding: None,
1947 message: "m\n".into(),
1948 raw_message: None,
1949 extra_headers: Vec::new(),
1950 });
1951 let commit = repo.odb().write(ObjectKind::Commit, &raw).expect("commit");
1952 crate::refs::write_ref(&repo.git_dir(), "refs/heads/main", &commit).expect("ref");
1953 (dir, repo, commit)
1954 }
1955
1956 fn reseal_graph(bytes: &mut Vec<u8>, algo: crate::objects::HashAlgo) {
1957 let hash_len = algo.len();
1958 let body_len = bytes.len().saturating_sub(hash_len);
1959 let digest = algo.checksum_digest(&bytes[..body_len]);
1960 bytes.truncate(body_len);
1961 bytes.extend_from_slice(&digest);
1962 }
1963
1964 fn memory_primary_over(
1965 dir: &tempfile::TempDir,
1966 files: &Repository,
1967 graph: Option<std::sync::Arc<super::CommitGraphChain>>,
1968 ) -> Repository {
1969 use crate::odb::store::{MemoryStore, ObjectStore, WritableObjectStore};
1970 let store = std::sync::Arc::new(MemoryStore::new(files.odb().hash_algo()));
1971 let source = files.odb().primary().expect("files primary");
1972 let mut oids = Vec::new();
1973 source
1974 .for_each_object(&mut |oid| {
1975 oids.push(*oid);
1976 std::ops::ControlFlow::Continue(())
1977 })
1978 .expect("list objects");
1979 for oid in oids {
1980 let obj = source.read(&oid).expect("read").expect("present");
1981 store
1982 .write(obj.kind, &obj.data, crate::odb::WriteOptions::default())
1983 .expect("copy");
1984 }
1985 let git_dir = dir.path().join(".git");
1986 let mut builder = crate::odb::OdbBuilder::files(git_dir.join("objects"))
1987 .primary(store)
1988 .alternates(false);
1989 if let Some(graph) = graph {
1990 builder = builder.commit_graph(graph);
1991 }
1992 crate::repo::RepositoryBuilder::new(crate::environment::RepositoryOptions::empty())
1993 .odb_builder(builder)
1994 .open(&git_dir, Some(dir.path()))
1995 .expect("open custom")
1996 }
1997
1998 #[test]
1999 fn from_bytes_parses_a_written_graph_and_rejects_bad_files() {
2000 let (_dir, repo, commit) = one_commit_repo();
2001 repo.write_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2002 .expect("write graph");
2003 let bytes =
2004 std::fs::read(repo.odb().objects_dir().join("info/commit-graph")).expect("graph file");
2005
2006 let chain = super::CommitGraphChain::from_bytes(bytes.clone()).expect("parse");
2007 assert_eq!(chain.total_commits(), 1);
2008 assert_eq!(chain.hash_algo(), crate::objects::HashAlgo::Sha1);
2009 assert_eq!(chain.graph_commit(&commit).map(|(p, _)| p), Some(vec![]));
2010
2011 let mut flipped = bytes.clone();
2012 let middle = flipped.len() / 2;
2013 flipped[middle] ^= 0xff;
2014 assert!(matches!(
2015 super::CommitGraphChain::from_bytes(flipped),
2016 Err(crate::error::Error::CorruptObject(_))
2017 ));
2018 assert!(super::CommitGraphChain::from_bytes(bytes[..20].to_vec()).is_err());
2019 assert!(super::CommitGraphChain::from_bytes(Vec::new()).is_err());
2020
2021 let mut split_layer = bytes;
2022 split_layer[7] = 1;
2023 reseal_graph(&mut split_layer, crate::objects::HashAlgo::Sha1);
2024 assert!(matches!(
2025 super::CommitGraphChain::from_bytes(split_layer),
2026 Err(crate::error::Error::CorruptObject(_))
2027 ));
2028 }
2029
2030 #[test]
2031 fn from_bytes_rejects_octopus_parents_without_an_edge_chunk() {
2032 let (_dir, repo, root) = one_commit_repo();
2033 let tree = repo.odb().read_commit(&root).expect("root commit").tree;
2034 let commit = |parents: Vec<ObjectId>, message: &str| {
2035 let raw = serialize_commit(&CommitData {
2036 tree,
2037 parents,
2038 author: "T <t@example.com> 200 +0000".into(),
2039 committer: "T <t@example.com> 200 +0000".into(),
2040 author_raw: Vec::new(),
2041 committer_raw: Vec::new(),
2042 encoding: None,
2043 message: message.into(),
2044 raw_message: None,
2045 extra_headers: Vec::new(),
2046 });
2047 repo.odb().write(ObjectKind::Commit, &raw).expect("commit")
2048 };
2049 let side_a = commit(vec![], "a\n");
2050 let side_b = commit(vec![], "b\n");
2051 let octopus = commit(vec![root, side_a, side_b], "octopus\n");
2052 crate::refs::write_ref(&repo.git_dir(), "refs/heads/main", &octopus).expect("ref");
2053 repo.write_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2054 .expect("write graph");
2055 let bytes =
2056 std::fs::read(repo.odb().objects_dir().join("info/commit-graph")).expect("graph file");
2057 let chain = super::CommitGraphChain::from_bytes(bytes.clone()).expect("valid graph");
2058 assert_eq!(chain.total_commits(), 4);
2059 assert_eq!(
2060 chain
2061 .graph_commit(&octopus)
2062 .map(|(parents, _)| parents.len()),
2063 Some(3)
2064 );
2065
2066 let mut no_edge_chunk = bytes;
2067 let edge_entry = (0..no_edge_chunk[6] as usize)
2068 .map(|i| 8 + i * 12)
2069 .find(|&e| no_edge_chunk[e..e + 4] == super::CHUNK_EXTRA_EDGES.to_be_bytes())
2070 .expect("octopus graph has an EDGE chunk");
2071 no_edge_chunk[edge_entry..edge_entry + 4].copy_from_slice(b"EDGX");
2072 reseal_graph(&mut no_edge_chunk, crate::objects::HashAlgo::Sha1);
2073 assert!(matches!(
2074 super::CommitGraphChain::from_bytes(no_edge_chunk),
2075 Err(crate::error::Error::CorruptObject(_))
2076 ));
2077 }
2078
2079 #[test]
2080 fn custom_primary_ignores_graph_files_and_uses_the_attached_graph() {
2081 let (dir, files, commit) = one_commit_repo();
2082 files
2083 .write_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2084 .expect("write graph");
2085 assert!(files.odb().commit_graph_chain().is_some());
2086
2087 let custom = memory_primary_over(&dir, &files, None);
2088 assert!(
2089 custom.odb().commit_graph_chain().is_none(),
2090 "a graph file under objects/ does not describe a custom store"
2091 );
2092
2093 let crate::maintenance::CommitGraphBuild::Built(built) = custom
2094 .build_commit_graph(&crate::maintenance::CommitGraphOptions::default())
2095 .expect("build")
2096 else {
2097 panic!("plain repository must not skip the graph");
2098 };
2099 assert_eq!(built.commit_count, 1);
2100 assert_eq!(built.bloom_stats, None);
2101 assert_eq!(
2102 built.bytes,
2103 std::fs::read(files.odb().objects_dir().join("info/commit-graph")).expect("file"),
2104 "the in-memory build matches the files writer byte for byte"
2105 );
2106
2107 let chain = std::sync::Arc::new(built.to_chain().expect("chain"));
2108 let attached = memory_primary_over(&dir, &files, Some(std::sync::Arc::clone(&chain)));
2109 let used = attached.odb().commit_graph_chain().expect("attached graph");
2110 assert!(std::sync::Arc::ptr_eq(&used, &chain));
2111 assert!(used.graph_commit(&commit).is_some());
2112 }
2113
2114 #[test]
2115 fn try_parse_rejects_bad_signature_and_truncated_toc() {
2116 let (_dir, repo, commit) = one_commit_repo();
2117 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2118 let mut infos = crate::objects::oid_hash_map();
2119 infos.insert(commit, info);
2120 let bloom = BloomFilterSettings::default();
2121 let (mut bytes, _) = build_commit_graph_bytes(
2122 &[commit],
2123 &infos_std(&infos),
2124 &repo.odb(),
2125 false,
2126 &bloom,
2127 None,
2128 &[],
2129 None,
2130 &empty_byte_filters(),
2131 &empty_byte_filters(),
2132 true,
2133 )
2134 .expect("write");
2135 let algo = repo.odb().hash_algo();
2136 bytes[0] = 0;
2137 reseal_graph(&mut bytes, algo);
2138 let err = super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes.clone())
2139 .unwrap_err();
2140 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2141 bytes.truncate(40);
2142 let err =
2143 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes).unwrap_err();
2144 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2145 let (mut good, _) = build_commit_graph_bytes(
2146 &[commit],
2147 &infos_std(&infos),
2148 &repo.odb(),
2149 false,
2150 &bloom,
2151 None,
2152 &[],
2153 None,
2154 &empty_byte_filters(),
2155 &empty_byte_filters(),
2156 true,
2157 )
2158 .expect("write");
2159 good[4] = 2;
2160 reseal_graph(&mut good, algo);
2161 let err =
2162 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), good).unwrap_err();
2163 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2164 let (mut checksum_bad, _) = build_commit_graph_bytes(
2165 &[commit],
2166 &infos_std(&infos),
2167 &repo.odb(),
2168 false,
2169 &bloom,
2170 None,
2171 &[],
2172 None,
2173 &empty_byte_filters(),
2174 &empty_byte_filters(),
2175 true,
2176 )
2177 .expect("write");
2178 let last = checksum_bad.len() - 1;
2179 checksum_bad[last] ^= 0xff;
2180 let err = super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), checksum_bad)
2181 .unwrap_err();
2182 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2183 }
2184
2185 #[test]
2186 fn parse_graph_file_returns_none_for_invalid_inputs() {
2187 let dir = tempfile::tempdir().expect("tempdir");
2188 let missing = dir.path().join("nope.graph");
2189 assert!(super::parse_graph_file(&missing).is_none());
2190 std::fs::write(dir.path().join("tiny.graph"), b"CGPH").expect("write");
2191 assert!(super::parse_graph_file(&dir.path().join("tiny.graph")).is_none());
2192 assert!(super::dump_bloom_filters(&missing).is_none());
2193 }
2194
2195 #[test]
2196 fn try_load_missing_graph_is_none() {
2197 let dir = tempfile::tempdir().expect("tempdir");
2198 let objects = dir.path().join("objects");
2199 std::fs::create_dir_all(objects.join("info")).expect("info");
2200 assert!(super::CommitGraphChain::try_load(&objects)
2201 .expect("ok")
2202 .is_none());
2203 }
2204
2205 #[test]
2206 fn try_load_across_finds_single_graph() {
2207 let (_dir, repo, _commit) = one_commit_repo();
2208 let objects = repo.odb().objects_dir();
2209 let sorted = vec![_commit];
2210 let info = load_commit_graph_commit_info(&repo.odb(), _commit).expect("info");
2211 let mut infos = crate::objects::oid_hash_map();
2212 infos.insert(_commit, info);
2213 let bloom = BloomFilterSettings::default();
2214 let (bytes, _) = build_commit_graph_bytes(
2215 &sorted,
2216 &infos_std(&infos),
2217 &repo.odb(),
2218 false,
2219 &bloom,
2220 None,
2221 &[],
2222 None,
2223 &empty_byte_filters(),
2224 &empty_byte_filters(),
2225 true,
2226 )
2227 .expect("write");
2228 std::fs::write(objects.join("info/commit-graph"), bytes).expect("write");
2229 let chain = super::CommitGraphChain::try_load_across(objects, &[])
2230 .expect("across")
2231 .expect("some");
2232 assert_eq!(chain.total_commits(), 1);
2233 }
2234
2235 #[test]
2236 fn try_parse_rejects_oidf_toc_offset_out_of_range() {
2237 let (_dir, repo, commit) = one_commit_repo();
2238 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2239 let mut infos = crate::objects::oid_hash_map();
2240 infos.insert(commit, info);
2241 let bloom = BloomFilterSettings::default();
2242 let (mut bytes, _) = build_commit_graph_bytes(
2243 &[commit],
2244 &infos_std(&infos),
2245 &repo.odb(),
2246 false,
2247 &bloom,
2248 None,
2249 &[],
2250 None,
2251 &empty_byte_filters(),
2252 &empty_byte_filters(),
2253 true,
2254 )
2255 .expect("write");
2256 let algo = repo.odb().hash_algo();
2257 let num_chunks = bytes[6] as usize;
2258 for i in 0..num_chunks {
2259 let e = 8 + i * 12;
2260 let id = u32::from_be_bytes(bytes[e..e + 4].try_into().unwrap());
2261 if id == 0x4f49_4446 {
2262 bytes[e + 4..e + 12].copy_from_slice(&u64::MAX.to_be_bytes());
2263 break;
2264 }
2265 }
2266 reseal_graph(&mut bytes, algo);
2267 let err =
2268 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes).unwrap_err();
2269 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2270 }
2271
2272 #[test]
2273 fn generation_overflow_without_gdo2_fails_parse() {
2274 let (_dir, repo, commit) = one_commit_repo();
2275 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2276 let mut infos = crate::objects::oid_hash_map();
2277 infos.insert(commit, info);
2278 let bloom = BloomFilterSettings::default();
2279 let (mut bytes, _) = build_commit_graph_bytes(
2280 &[commit],
2281 &infos_std(&infos),
2282 &repo.odb(),
2283 false,
2284 &bloom,
2285 None,
2286 &[],
2287 None,
2288 &empty_byte_filters(),
2289 &empty_byte_filters(),
2290 true,
2291 )
2292 .expect("write");
2293 if bytes.len() > 100 {
2294 for i in 80..120 {
2295 if i + 4 <= bytes.len() - 20 {
2296 bytes[i] = 0x80;
2297 bytes[i + 1] = 0x00;
2298 bytes[i + 2] = 0x00;
2299 bytes[i + 3] = 0x00;
2300 break;
2301 }
2302 }
2303 }
2304 let err =
2305 super::CommitGraphLayer::try_parse(std::path::PathBuf::from("g"), bytes).unwrap_err();
2306 assert!(matches!(err, crate::error::Error::CorruptObject(_)));
2307 }
2308
2309 #[test]
2310 fn bloom_precheck_not_in_graph_for_missing_oid() {
2311 let (_dir, repo, commit) = one_commit_repo();
2312 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2313 let mut infos = crate::objects::oid_hash_map();
2314 infos.insert(commit, info);
2315 let bloom = BloomFilterSettings::default();
2316 let (bytes, _) = build_commit_graph_bytes(
2317 &[commit],
2318 &infos_std(&infos),
2319 &repo.odb(),
2320 true,
2321 &bloom,
2322 None,
2323 &[],
2324 None,
2325 &empty_byte_filters(),
2326 &empty_byte_filters(),
2327 true,
2328 )
2329 .expect("write");
2330 std::fs::write(repo.odb().objects_dir().join("info/commit-graph"), bytes).expect("write");
2331 let chain = super::CommitGraphChain::load(repo.odb().objects_dir()).expect("chain");
2332 let missing = ObjectId::from_hex("1234567890123456789012345678901234567890").expect("oid");
2333 let pre = chain
2334 .bloom_precheck_for_paths(&repo.odb(), missing, &[], None, -1, true)
2335 .expect("precheck");
2336 assert_eq!(pre, super::BloomPrecheck::NotInGraph);
2337 }
2338
2339 #[test]
2340 fn bloom_precheck_respects_read_changed_paths_flag() {
2341 let (_dir, repo, commit) = one_commit_repo();
2342 let mut sorted = vec![commit];
2343 let info = load_commit_graph_commit_info(&repo.odb(), commit).expect("info");
2344 let mut infos = crate::objects::oid_hash_map();
2345 infos.insert(commit, info);
2346 let bloom = BloomFilterSettings::default();
2347 let (bytes, _) = build_commit_graph_bytes(
2348 &sorted,
2349 &infos_std(&infos),
2350 &repo.odb(),
2351 true,
2352 &bloom,
2353 None,
2354 &[],
2355 None,
2356 &empty_byte_filters(),
2357 &empty_byte_filters(),
2358 true,
2359 )
2360 .expect("write");
2361 std::fs::write(repo.odb().objects_dir().join("info/commit-graph"), bytes).expect("write");
2362 let chain = super::CommitGraphChain::load(repo.odb().objects_dir()).expect("chain");
2363 let pre = chain
2364 .bloom_precheck_for_paths(&repo.odb(), commit, &["f.txt".to_owned()], None, -1, false)
2365 .expect("precheck");
2366 assert_eq!(pre, super::BloomPrecheck::Inapplicable);
2367 sorted.clear();
2368 }
2369
2370 #[test]
2371 fn bloom_skips_most_steps_when_tracked_path_unchanged_in_linear_history() {
2372 use crate::bloom::bloom_keyvec_for_path;
2373 use crate::commit_graph_write::{build_commit_graph_bytes, load_commit_graph_commit_info};
2374 use crate::index::{Index, IndexEntry, MODE_REGULAR};
2375 use crate::objects::{serialize_commit, CommitData, ObjectKind};
2376 use crate::pathspec::bloom_lookup_prefix_with_cwd;
2377 use crate::repo::init_repository;
2378 use crate::write_tree::write_tree_from_index;
2379 use crate::RefStorageFormat;
2380 use std::fs;
2381
2382 let dir = tempfile::tempdir().expect("tempdir");
2383 let repo = init_repository(dir.path(), false, "main", None, RefStorageFormat::Files)
2384 .expect("init");
2385 let path = "tracked.txt";
2386 let stable = b"stable payload\n";
2387 let blob = repo.odb().write(ObjectKind::Blob, stable).expect("blob");
2388 let mut parent: Option<ObjectId> = None;
2389 let mut oids = Vec::new();
2390 for i in 0..24usize {
2391 let mut index = Index::new();
2392 index.hash_algo = repo.odb().hash_algo();
2393 index.add_or_replace(IndexEntry {
2394 ctime_sec: 0,
2395 ctime_nsec: 0,
2396 mtime_sec: 0,
2397 mtime_nsec: 0,
2398 dev: 0,
2399 ino: 0,
2400 mode: MODE_REGULAR,
2401 uid: 0,
2402 gid: 0,
2403 size: stable.len() as u32,
2404 oid: blob,
2405 flags: 4,
2406 flags_extended: None,
2407 path: path.as_bytes().to_vec(),
2408 base_index_pos: 0,
2409 });
2410 if i % 5 == 3 {
2411 let noise = repo
2412 .odb()
2413 .write(ObjectKind::Blob, format!("noise {i}\n").as_bytes())
2414 .expect("noise blob");
2415 index.add_or_replace(IndexEntry {
2416 ctime_sec: 0,
2417 ctime_nsec: 0,
2418 mtime_sec: 0,
2419 mtime_nsec: 0,
2420 dev: 0,
2421 ino: 0,
2422 mode: MODE_REGULAR,
2423 uid: 0,
2424 gid: 0,
2425 size: 8,
2426 oid: noise,
2427 flags: 4,
2428 flags_extended: None,
2429 path: format!("other{i}.txt").into_bytes(),
2430 base_index_pos: 0,
2431 });
2432 }
2433 let tree = write_tree_from_index(&repo.odb(), &index, "").expect("tree");
2434 let mut parents = Vec::new();
2435 if let Some(p) = parent {
2436 parents.push(p);
2437 }
2438 let raw = serialize_commit(&CommitData {
2439 tree,
2440 parents,
2441 author: "T <t@example.com> 100 +0000".into(),
2442 committer: "T <t@example.com> 100 +0000".into(),
2443 author_raw: Vec::new(),
2444 committer_raw: Vec::new(),
2445 encoding: None,
2446 message: format!("c{i}\n"),
2447 raw_message: None,
2448 extra_headers: Vec::new(),
2449 });
2450 let commit = repo.odb().write(ObjectKind::Commit, &raw).expect("commit");
2451 oids.push(commit);
2452 parent = Some(commit);
2453 }
2454 let head = *oids.last().expect("head");
2455 let mut infos = crate::objects::oid_hash_map();
2456 for oid in &oids {
2457 infos.insert(
2458 *oid,
2459 load_commit_graph_commit_info(&repo.odb(), *oid).expect("info"),
2460 );
2461 }
2462 let bloom = BloomFilterSettings::default();
2463 let (bytes, _) = build_commit_graph_bytes(
2464 &oids,
2465 &infos_std(&infos),
2466 &repo.odb(),
2467 true,
2468 &bloom,
2469 None,
2470 &[],
2471 None,
2472 &empty_byte_filters(),
2473 &empty_byte_filters(),
2474 true,
2475 )
2476 .expect("graph bytes");
2477 fs::write(repo.odb().objects_dir().join("info/commit-graph"), bytes).expect("write graph");
2478 let chain = super::CommitGraphChain::load(repo.odb().objects_dir()).expect("chain");
2479 let settings = chain.top_layer_bloom_settings().expect("bloom");
2480 let norm = bloom_lookup_prefix_with_cwd(path, None).expect("prefix");
2481 let keys = bloom_keyvec_for_path(norm.as_str(), &settings);
2482 let mut current = head;
2483 let mut bloom_yes = 0usize;
2484 let mut bloom_no = 0usize;
2485 while bloom_yes + bloom_no < 10_000 {
2486 let Some((parents, _)) = chain.graph_commit(¤t) else {
2487 break;
2488 };
2489 let Some(parent) = parents.first().copied() else {
2490 break;
2491 };
2492 if chain.bloom_definitely_not_for_keys(current, keys.as_slice()) {
2493 bloom_yes += 1;
2494 } else {
2495 bloom_no += 1;
2496 }
2497 current = parent;
2498 }
2499 assert!(
2500 bloom_yes > bloom_no,
2501 "expected bloom to skip unchanged tracked path (yes={bloom_yes} no={bloom_no})"
2502 );
2503 }
2504
2505 #[test]
2506 #[ignore = "local git.git worktree; GRIT_BLAME_BENCH_REPO"]
2507 fn first_parent_makefile_bloom_skip_rate_on_git_dot_git() {
2508 use crate::bloom::bloom_keyvec_for_path;
2509 use crate::environment::RepositoryOptions;
2510 use crate::pathspec::bloom_lookup_prefix_with_cwd;
2511 use crate::repo::Repository;
2512
2513 let repo_path = std::env::var("GRIT_BLAME_BENCH_REPO")
2514 .unwrap_or_else(|_| "/tmp/grit-bench/git-dot-git-worktree".to_string());
2515 if !std::path::Path::new(&repo_path).join(".git").exists() {
2516 return;
2517 }
2518 let git_dir = std::path::Path::new(&repo_path).join(".git");
2519 let repo =
2520 Repository::open(git_dir.as_path(), &RepositoryOptions::default()).expect("open");
2521 let chain = repo.odb().commit_graph_chain().expect("graph");
2522 let settings = chain.top_layer_bloom_settings().expect("bloom");
2523 let norm = bloom_lookup_prefix_with_cwd("Makefile", None).expect("prefix");
2524 let keys = bloom_keyvec_for_path(norm.as_str(), &settings);
2525 let head = *crate::state::resolve_head(repo.git_dir())
2526 .expect("head")
2527 .oid()
2528 .expect("commit");
2529 let mut current = head;
2530 let mut bloom_yes = 0usize;
2531 let mut bloom_no = 0usize;
2532 for _ in 0..10_000 {
2533 let Some((parents, _)) = chain.graph_commit(¤t) else {
2534 break;
2535 };
2536 let Some(parent) = parents.first().copied() else {
2537 break;
2538 };
2539 if chain.bloom_definitely_not_for_keys(current, keys.as_slice()) {
2540 bloom_yes += 1;
2541 } else {
2542 bloom_no += 1;
2543 }
2544 current = parent;
2545 }
2546 assert!(
2547 bloom_yes > bloom_no,
2548 "expected mostly bloom skips, yes={bloom_yes} no={bloom_no}"
2549 );
2550 }
2551
2552 #[test]
2553 fn bloom_definitely_not_when_directory_key_present_but_not_full_path() {
2554 use crate::bloom::{
2555 bloom_filter_contains, bloom_keyvec_for_path, build_bloom_filter_data,
2556 collect_changed_paths_for_bloom,
2557 };
2558
2559 let bloom = BloomFilterSettings::default();
2560 let changed = collect_changed_paths_for_bloom(&["builtin/sibling.c".to_owned()]);
2561 let (filter, _) = build_bloom_filter_data(&changed, 1, &bloom);
2562 let path_keys = bloom_keyvec_for_path("builtin/commit.c", &bloom);
2563 assert!(path_keys.len() >= 2);
2564 assert_eq!(
2565 bloom_filter_contains(&path_keys[0], &filter, &bloom),
2566 Ok(false),
2567 "full path key must be absent when only a sibling changed"
2568 );
2569 assert_eq!(
2570 bloom_filter_contains(&path_keys[1], &filter, &bloom),
2571 Ok(true),
2572 "directory prefix key should be present"
2573 );
2574 assert!(super::bloom_definitely_not_from_filter(
2575 &filter, &bloom, &path_keys
2576 ));
2577 }
2578
2579 #[cfg_attr(
2580 not(feature = "full-tests"),
2581 ignore = "starts system git or is heavy; run with --features full-tests"
2582 )]
2583 #[test]
2584 fn large_git_split_chain_layers_use_mapped_backing() {
2585 use std::process::Command;
2586
2587 let git_ok = Command::new("git")
2588 .arg("--version")
2589 .output()
2590 .map(|o| o.status.success())
2591 .unwrap_or(false);
2592 if !git_ok {
2593 eprintln!("SKIP: git unavailable");
2594 return;
2595 }
2596 let dir = tempfile::tempdir().expect("tempdir");
2597 let run = |args: &[&str]| {
2598 let out = Command::new("git")
2599 .current_dir(dir.path())
2600 .args(args)
2601 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2602 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2603 .env("GIT_AUTHOR_NAME", "T")
2604 .env("GIT_AUTHOR_EMAIL", "t@example.com")
2605 .env("GIT_COMMITTER_NAME", "T")
2606 .env("GIT_COMMITTER_EMAIL", "t@example.com")
2607 .output()
2608 .expect("git");
2609 assert!(out.status.success(), "git {args:?}");
2610 };
2611 run(&["init", "-q", "-b", "main"]);
2612 run(&["commit", "--allow-empty", "-q", "-m", "bootstrap"]);
2613 for i in 0..150 {
2614 run(&[
2615 "commit",
2616 "--allow-empty",
2617 "-q",
2618 "-m",
2619 &format!("base-fill-{i}"),
2620 ]);
2621 }
2622 run(&[
2623 "commit-graph",
2624 "write",
2625 "--reachable",
2626 "--split",
2627 "--max-commits",
2628 "40",
2629 ]);
2630 for i in 150..190 {
2631 run(&[
2632 "commit",
2633 "--allow-empty",
2634 "-q",
2635 "-m",
2636 &format!("tip-fill-{i}"),
2637 ]);
2638 }
2639 run(&[
2640 "commit-graph",
2641 "write",
2642 "--reachable",
2643 "--split",
2644 "--max-commits",
2645 "40",
2646 ]);
2647 let objects = dir.path().join(".git/objects");
2648 let chain = super::CommitGraphChain::try_load(&objects)
2649 .expect("load")
2650 .expect("chain");
2651 assert!(
2652 chain.num_layers() >= 2,
2653 "expected multi-layer chain, got {}",
2654 chain.num_layers()
2655 );
2656 let mut saw_mapped_large_layer = false;
2657 for path in chain.layer_paths_oldest_first() {
2658 let len = std::fs::metadata(&path).expect("meta").len();
2659 if len <= 8192 {
2660 continue;
2661 }
2662 let storage = crate::pack_map::PackData::open_with_min_len(
2663 &path,
2664 super::COMMIT_GRAPH_MIN_FILE_LEN,
2665 )
2666 .expect("open layer");
2667 assert!(
2668 storage.is_mapped_backing(),
2669 "expected mmap backing for {} ({} bytes)",
2670 path.display(),
2671 len
2672 );
2673 saw_mapped_large_layer = true;
2674 }
2675 assert!(
2676 saw_mapped_large_layer,
2677 "expected at least one chain layer larger than 8192 bytes to use mmap"
2678 );
2679 }
2680}