grit_lib/diff/
tree_content.rs

1//! Parallel blob loads for tree-to-tree diffs (commit ranges, `grit diff A B`).
2
3use crate::config::ConfigSet;
4use crate::diff_rename_parallel::parallel_map_indices;
5use crate::error::{Error, Result};
6use crate::objects::pack_store::PackReadContextGuard;
7use crate::objects::ObjectId;
8use crate::odb::Odb;
9use crate::parallel::run_ordered_batches;
10use crate::parallel::{try_par_map_with_local, ParallelHashError};
11use crate::parallel::{ParallelOp, ParallelOptions, ThreadPolicy};
12use std::num::NonZeroUsize;
13use std::sync::atomic::{AtomicUsize, Ordering};
14use std::sync::Arc;
15
16use super::detail::{line_change_counts, LineStat};
17use super::structured::{structured_file_diff, SideBytes, StructuredFileDiff};
18use super::DiffEntry;
19use super::DiffOptions;
20
21fn note_blob_read(counter: Option<&AtomicUsize>) {
22    if let Some(c) = counter {
23        c.fetch_add(1, Ordering::Relaxed);
24    }
25}
26
27/// Pool options for per-entry content work that should fan out whenever it can.
28///
29/// A single entry always runs on the calling thread, so the policy (which probes the
30/// CPU count and cgroup quota) is only consulted when there is more than one.
31fn per_entry_parallel_options(
32    thread_policy: &ThreadPolicy,
33    config: &ConfigSet,
34    entry_count: usize,
35) -> ParallelOptions {
36    if entry_count < 2 {
37        return ParallelOptions {
38            threads: NonZeroUsize::MIN,
39            min_items: 0,
40            min_total_bytes: None,
41            signed_entry_threshold: None,
42        };
43    }
44    let mut par_opts = thread_policy.resolve(ParallelOp::DiffContent, config);
45    par_opts.min_items = 0;
46    par_opts.min_total_bytes = None;
47    par_opts
48}
49
50/// Tree diff entry with structured line hunks ready for CLI rendering.
51#[derive(Debug, Clone, PartialEq, Eq)]
52pub struct TreeDiffStructured {
53    /// Tree diff metadata (paths, modes, oids, status).
54    pub entry: DiffEntry,
55    /// Structured hunks and mode metadata.
56    pub structured: StructuredFileDiff,
57    /// `true` when either side looks like a binary blob (NUL in the first 8 KiB).
58    pub binary: bool,
59    /// Worktree/index side bytes when content is not materialized in the ODB.
60    pub patch_bytes: Option<(SideBytes, SideBytes)>,
61}
62
63/// One tree diff entry with both sides loaded for line-level diffing.
64#[derive(Debug, Clone, PartialEq, Eq)]
65pub struct TreeDiffContent {
66    /// Tree diff metadata (paths, modes, oids, status).
67    pub entry: DiffEntry,
68    /// Old-side bytes prepared for [`super::structured::structured_text_diff`].
69    pub old_bytes: SideBytes,
70    /// New-side bytes prepared for structured diffing.
71    pub new_bytes: SideBytes,
72    /// `true` when either side looks like a binary blob (NUL in the first 8 KiB).
73    pub binary: bool,
74}
75
76/// Load old/new blob bytes for each entry, using a worker pool when thresholds allow.
77///
78/// Submodule gitlinks are synthesized as `Subproject commit <oid>` lines; missing sides
79/// are empty. Order matches `entries`.
80///
81/// # Errors
82///
83/// Propagates object database read failures.
84pub fn tree_diff_contents_parallel(
85    odb: &Odb,
86    entries: Vec<DiffEntry>,
87    config: &ConfigSet,
88    thread_policy: &ThreadPolicy,
89) -> Result<Vec<TreeDiffContent>> {
90    if entries.is_empty() {
91        return Ok(Vec::new());
92    }
93    let mut par_opts = thread_policy.resolve(ParallelOp::DiffContent, config);
94    par_opts.min_items = par_opts.min_items.min(8);
95    let est_bytes = entries.len().saturating_mul(8192);
96    let rows = parallel_map_indices(odb, entries.len(), par_opts, est_bytes, |i| {
97        let entry = &entries[i];
98        let (old_bytes, old_bin) = load_old_side(odb, entry, None)?;
99        let (new_bytes, new_bin) = load_new_side(odb, entry, None)?;
100        Ok::<TreeDiffContent, Error>(TreeDiffContent {
101            entry: entry.clone(),
102            old_bytes,
103            new_bytes,
104            binary: old_bin || new_bin,
105        })
106    })?;
107    Ok(rows)
108}
109
110/// Per-file line counts for stat modes (loads blobs, no hunk allocation).
111///
112/// # Errors
113///
114/// Propagates object database read failures.
115pub fn tree_diff_stat_parallel(
116    odb: &Odb,
117    entries: Vec<DiffEntry>,
118    config: &ConfigSet,
119    thread_policy: &ThreadPolicy,
120    diff_options: DiffOptions,
121) -> Result<Vec<(DiffEntry, bool, Option<LineStat>)>> {
122    if entries.is_empty() {
123        return Ok(Vec::new());
124    }
125    let par_opts = per_entry_parallel_options(thread_policy, config, entries.len());
126    let line_diff_opts = diff_options;
127    let blob_counter = line_diff_opts.blob_read_counter.clone();
128    let store = odb.pack_store_for(odb.objects_dir());
129    let rows = try_par_map_with_local(
130        &entries,
131        par_opts,
132        0,
133        || PackReadContextGuard::install(Arc::clone(&store)),
134        move |_ctx, entry| {
135            let blob_reads = blob_counter.as_deref();
136            if !entry.old_oid.is_zero()
137                && entry.old_oid == entry.new_oid
138                && entry.old_mode == entry.new_mode
139            {
140                let (_, old_bin) = load_old_side(odb, entry, blob_reads)?;
141                let binary = old_bin;
142                return Ok((
143                    entry.clone(),
144                    binary,
145                    if binary {
146                        None
147                    } else {
148                        Some(LineStat::default())
149                    },
150                ));
151            }
152            let (old_bytes, old_bin) = load_old_side(odb, entry, blob_reads)?;
153            let (new_bytes, new_bin) = load_new_side(odb, entry, blob_reads)?;
154            let binary = old_bin || new_bin;
155            let stat = if binary {
156                None
157            } else {
158                Some(line_change_counts(
159                    &old_bytes.text(),
160                    &new_bytes.text(),
161                    &line_diff_opts,
162                ))
163            };
164            Ok((entry.clone(), binary, stat))
165        },
166    );
167    match rows {
168        Ok(rows) => Ok(rows),
169        Err(ParallelHashError::Task(err)) => Err(err),
170    }
171}
172
173/// Load sides and compute a structured line diff for one tree diff entry.
174///
175/// # Errors
176///
177/// Propagates object database read failures.
178pub fn tree_diff_structured_entry(
179    odb: &Odb,
180    entry: &DiffEntry,
181    diff_options: &DiffOptions,
182) -> Result<TreeDiffStructured> {
183    let blob_reads = diff_options.blob_read_counter.as_deref();
184    if !entry.old_oid.is_zero()
185        && entry.old_oid == entry.new_oid
186        && entry.old_mode == entry.new_mode
187    {
188        let (old_bytes, old_bin) = load_old_side(odb, entry, blob_reads)?;
189        return Ok(TreeDiffStructured {
190            entry: entry.clone(),
191            structured: StructuredFileDiff {
192                old_mode: mode_display(&entry.old_mode),
193                new_mode: mode_display(&entry.new_mode),
194                old_encoding_lossy: old_bytes.encoding_lossy,
195                new_encoding_lossy: old_bytes.encoding_lossy,
196                hunks: Vec::new(),
197            },
198            binary: old_bin,
199            patch_bytes: None,
200        });
201    }
202    let (old_bytes, old_bin) = load_old_side(odb, entry, blob_reads)?;
203    let (new_bytes, new_bin) = load_new_side(odb, entry, blob_reads)?;
204    let binary = old_bin || new_bin;
205    let structured = if binary {
206        StructuredFileDiff {
207            old_mode: mode_display(&entry.old_mode),
208            new_mode: mode_display(&entry.new_mode),
209            old_encoding_lossy: old_bytes.encoding_lossy,
210            new_encoding_lossy: new_bytes.encoding_lossy,
211            hunks: Vec::new(),
212        }
213    } else {
214        structured_file_diff(
215            &entry.old_mode,
216            &entry.new_mode,
217            &old_bytes,
218            &new_bytes,
219            diff_options.clone(),
220        )
221    };
222    Ok(TreeDiffStructured {
223        entry: entry.clone(),
224        structured,
225        binary,
226        patch_bytes: None,
227    })
228}
229
230/// Visit structured tree diffs in path order without retaining every file's hunks at once.
231///
232/// When thread policy allows, workers compute up to about `2 × threads` entries ahead while
233/// the callback consumes results strictly in order.
234///
235/// # Errors
236///
237/// Propagates object database read failures and callback errors.
238pub fn for_each_tree_diff_structured(
239    odb: &Odb,
240    entries: &[DiffEntry],
241    diff_options: DiffOptions,
242    config: &ConfigSet,
243    thread_policy: &ThreadPolicy,
244    mut visit: impl FnMut(TreeDiffStructured) -> Result<()>,
245) -> Result<()> {
246    if entries.is_empty() {
247        return Ok(());
248    }
249    let diff_options = diff_options.cap_word_level_inline_for_changed_files(entries.len());
250    let par_opts = per_entry_parallel_options(thread_policy, config, entries.len());
251    let n_workers = par_opts.threads.get();
252    if n_workers <= 1 || entries.len() < 2 {
253        for entry in entries {
254            visit(tree_diff_structured_entry(odb, entry, &diff_options)?)?;
255        }
256        return Ok(());
257    }
258    let line_diff_opts = diff_options;
259    let store = odb.pack_store_for(odb.objects_dir());
260    run_ordered_batches(
261        entries,
262        1,
263        n_workers,
264        || PackReadContextGuard::install(Arc::clone(&store)),
265        move |_ctx, entry| tree_diff_structured_entry(odb, entry, &line_diff_opts),
266        move |_batch, rows| {
267            for row in rows {
268                visit(row)?;
269            }
270            Ok(())
271        },
272    )
273}
274
275/// Load sides and compute structured line diffs for each entry in parallel.
276///
277/// # Errors
278///
279/// Propagates object database read failures.
280pub fn tree_diff_structured_parallel(
281    odb: &Odb,
282    entries: Vec<DiffEntry>,
283    config: &ConfigSet,
284    thread_policy: &ThreadPolicy,
285    diff_options: DiffOptions,
286) -> Result<Vec<TreeDiffStructured>> {
287    if entries.is_empty() {
288        return Ok(Vec::new());
289    }
290    let diff_options = diff_options.cap_word_level_inline_for_changed_files(entries.len());
291    let par_opts = per_entry_parallel_options(thread_policy, config, entries.len());
292    let line_diff_opts = diff_options;
293    let store = odb.pack_store_for(odb.objects_dir());
294    let rows = try_par_map_with_local(
295        &entries,
296        par_opts,
297        0,
298        || PackReadContextGuard::install(Arc::clone(&store)),
299        move |_ctx, entry| tree_diff_structured_entry(odb, entry, &line_diff_opts),
300    );
301    match rows {
302        Ok(rows) => Ok(rows),
303        Err(ParallelHashError::Task(err)) => Err(err),
304    }
305}
306
307fn mode_display(mode: &str) -> Option<String> {
308    if mode == "000000" {
309        None
310    } else {
311        Some(mode.to_owned())
312    }
313}
314
315fn load_old_side(
316    odb: &Odb,
317    e: &DiffEntry,
318    blob_reads: Option<&AtomicUsize>,
319) -> Result<(SideBytes, bool)> {
320    if e.old_mode == "160000" {
321        return Ok((
322            SideBytes::from_bytes(gitlink_line(&e.old_oid).into_bytes()),
323            false,
324        ));
325    }
326    if e.old_mode == "000000" {
327        return Ok((SideBytes::from_bytes(Vec::new()), false));
328    }
329    blob_bytes(odb, &e.old_oid, blob_reads)
330}
331
332fn load_new_side(
333    odb: &Odb,
334    e: &DiffEntry,
335    blob_reads: Option<&AtomicUsize>,
336) -> Result<(SideBytes, bool)> {
337    if e.new_mode == "160000" {
338        return Ok((
339            SideBytes::from_bytes(gitlink_line(&e.new_oid).into_bytes()),
340            false,
341        ));
342    }
343    if e.new_mode == "000000" {
344        return Ok((SideBytes::from_bytes(Vec::new()), false));
345    }
346    blob_bytes(odb, &e.new_oid, blob_reads)
347}
348
349fn gitlink_line(oid: &ObjectId) -> String {
350    if *oid == ObjectId::zero() {
351        return String::new();
352    }
353    format!("Subproject commit {}\n", oid.to_hex())
354}
355
356fn blob_bytes(
357    odb: &Odb,
358    oid: &ObjectId,
359    blob_reads: Option<&AtomicUsize>,
360) -> Result<(SideBytes, bool)> {
361    if *oid == ObjectId::zero() {
362        return Ok((SideBytes::from_bytes(Vec::new()), false));
363    }
364    note_blob_read(blob_reads);
365    let data = odb.read(oid)?.data;
366    Ok(bytes_for_diff(&data))
367}
368
369fn bytes_for_diff(data: &[u8]) -> (SideBytes, bool) {
370    if data.iter().take(8000).any(|&b| b == 0) {
371        (SideBytes::from_bytes(Vec::new()), true)
372    } else {
373        (SideBytes::from_bytes(data.to_vec()), false)
374    }
375}
376
377#[cfg(test)]
378mod tests {
379    use super::*;
380    use crate::config::ConfigSet;
381    use crate::diff::{diff_trees, DiffOptions, DiffStatus};
382    use crate::objects::{serialize_tree, ObjectKind, TreeEntry};
383    use crate::parallel::ThreadPolicy;
384    use crate::repo::{InitOptions, Repository};
385
386    fn init_repo() -> (tempfile::TempDir, Repository) {
387        let dir = tempfile::tempdir().expect("tempdir");
388        let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
389        (dir, repo)
390    }
391
392    #[test]
393    fn tree_diff_contents_parallel_matches_serial_load() {
394        let (_dir, repo) = init_repo();
395        let odb = repo.odb();
396        let a = odb.write(ObjectKind::Blob, b"alpha\n").expect("write a");
397        let b = odb.write(ObjectKind::Blob, b"beta\n").expect("write b");
398        let tree_a = odb
399            .write(
400                ObjectKind::Tree,
401                &serialize_tree(&[TreeEntry {
402                    mode: 0o100644,
403                    name: b"file-a".to_vec(),
404                    oid: a,
405                }]),
406            )
407            .expect("tree a");
408        let tree_b = odb
409            .write(
410                ObjectKind::Tree,
411                &serialize_tree(&[TreeEntry {
412                    mode: 0o100644,
413                    name: b"file-b".to_vec(),
414                    oid: b,
415                }]),
416            )
417            .expect("tree b");
418        let raw = diff_trees(odb, Some(&tree_a), Some(&tree_b), "").expect("diff");
419        assert_eq!(raw.len(), 2);
420        let config = repo.config().expect("config");
421        let serial = tree_diff_contents_parallel(
422            odb,
423            raw.clone(),
424            &config,
425            &ThreadPolicy::with_threads_override(Some(1)),
426        )
427        .expect("serial load");
428        let parallel = tree_diff_contents_parallel(
429            odb,
430            raw,
431            &config,
432            &ThreadPolicy::with_threads_override(Some(4)),
433        )
434        .expect("parallel load");
435        assert_eq!(serial, parallel);
436        assert!(serial
437            .iter()
438            .any(|c| c.entry.status == DiffStatus::Added || c.entry.status == DiffStatus::Deleted));
439    }
440
441    #[test]
442    fn name_only_does_not_read_blobs_for_tree_to_tree() {
443        use crate::config::ConfigSet;
444        use crate::diff::{build_file_diffs, DiffDetail, DiffOptions};
445
446        let (_dir, repo) = init_repo();
447        let odb = repo.odb();
448        let a = odb.write(ObjectKind::Blob, b"alpha\n").expect("write a");
449        let b = odb.write(ObjectKind::Blob, b"beta\n").expect("write b");
450        let old = odb
451            .write(
452                ObjectKind::Tree,
453                &serialize_tree(&[TreeEntry {
454                    mode: 0o100644,
455                    name: b"f.txt".to_vec(),
456                    oid: a,
457                }]),
458            )
459            .expect("old tree");
460        let new = odb
461            .write(
462                ObjectKind::Tree,
463                &serialize_tree(&[TreeEntry {
464                    mode: 0o100644,
465                    name: b"f.txt".to_vec(),
466                    oid: b,
467                }]),
468            )
469            .expect("new tree");
470        let entries = diff_trees(odb, Some(&old), Some(&new), "").expect("entries");
471        let metrics = odb.hot_path_metrics();
472        metrics.set_blob_counting(true);
473        metrics.reset_blob_content_reads();
474        let config = ConfigSet::new();
475        let _ = build_file_diffs(
476            odb,
477            entries,
478            DiffDetail::NameOnly,
479            DiffOptions::default(),
480            &config,
481            &ThreadPolicy::default(),
482        )
483        .expect("name-only");
484        metrics.set_blob_counting(false);
485        assert_eq!(metrics.blob_content_reads(), 0);
486    }
487
488    #[test]
489    fn single_entry_runs_serially_without_consulting_the_policy() {
490        let config = ConfigSet::new();
491        let policy = ThreadPolicy::with_threads_override(Some(4));
492        for count in [0, 1] {
493            let opts = per_entry_parallel_options(&policy, &config, count);
494            assert_eq!(opts.threads, NonZeroUsize::MIN);
495        }
496        let opts = per_entry_parallel_options(&policy, &config, 2);
497        assert_eq!(opts.min_items, 0);
498        assert_eq!(opts.min_total_bytes, None);
499        assert_eq!(
500            opts.threads,
501            policy.resolve(ParallelOp::DiffContent, &config).threads
502        );
503    }
504
505    #[test]
506    fn single_entry_structured_diff_matches_threaded_policy() {
507        let (_dir, repo) = init_repo();
508        let odb = repo.odb();
509        let a = odb.write(ObjectKind::Blob, b"one\ntwo\n").expect("write a");
510        let b = odb
511            .write(ObjectKind::Blob, b"one\nthree\n")
512            .expect("write b");
513        let tree = |oid| {
514            odb.write(
515                ObjectKind::Tree,
516                &serialize_tree(&[TreeEntry {
517                    mode: 0o100644,
518                    name: b"f.txt".to_vec(),
519                    oid,
520                }]),
521            )
522            .expect("tree")
523        };
524        let (old, new) = (tree(a), tree(b));
525        let entries = diff_trees(odb, Some(&old), Some(&new), "").expect("entries");
526        assert_eq!(entries.len(), 1);
527        let config = repo.config().expect("config");
528        let run = |threads| {
529            tree_diff_structured_parallel(
530                odb,
531                entries.clone(),
532                &config,
533                &ThreadPolicy::with_threads_override(Some(threads)),
534                DiffOptions::default(),
535            )
536            .expect("structured diff")
537        };
538        let serial = run(1);
539        assert_eq!(serial, run(4));
540        assert_eq!(serial[0].structured.hunks.len(), 1);
541    }
542}
grit_lib/diff/tree_content.rs

grit_lib/diff/
tree_content.rs
