grit_lib/diff/
tree_content.rs

1//! Parallel blob loads for tree-to-tree diffs (commit ranges, `grit diff A B`).
2
3use crate::config::ConfigSet;
4use crate::diff_rename_parallel::parallel_map_indices;
5use crate::error::{Error, Result};
6use crate::objects::pack_store::PackReadContextGuard;
7use crate::objects::FileMode;
8use crate::objects::ObjectId;
9use crate::odb::Odb;
10use crate::parallel::run_ordered_batches;
11use crate::parallel::{try_par_map_with_local, ParallelHashError};
12use crate::parallel::{ParallelOp, ParallelOptions, ThreadPolicy};
13use std::num::NonZeroUsize;
14use std::sync::atomic::{AtomicUsize, Ordering};
15use std::sync::Arc;
16
17use super::detail::{line_change_counts, LineStat};
18use super::structured::{structured_file_diff, SideBytes, StructuredFileDiff};
19use super::DiffEntry;
20use super::DiffOptions;
21
22fn note_blob_read(counter: Option<&AtomicUsize>) {
23    if let Some(c) = counter {
24        c.fetch_add(1, Ordering::Relaxed);
25    }
26}
27
28/// Pool options for per-entry content work that should fan out whenever it can.
29///
30/// A single entry always runs on the calling thread, so the policy (which probes the
31/// CPU count and cgroup quota) is only consulted when there is more than one.
32fn per_entry_parallel_options(
33    thread_policy: &ThreadPolicy,
34    config: &ConfigSet,
35    entry_count: usize,
36) -> ParallelOptions {
37    if entry_count < 2 {
38        return ParallelOptions {
39            threads: NonZeroUsize::MIN,
40            min_items: 0,
41            min_total_bytes: None,
42            signed_entry_threshold: None,
43        };
44    }
45    let mut par_opts = thread_policy.resolve(ParallelOp::DiffContent, config);
46    par_opts.min_items = 0;
47    par_opts.min_total_bytes = None;
48    par_opts
49}
50
51/// Tree diff entry with structured line hunks ready for CLI rendering.
52#[derive(Debug, Clone, PartialEq, Eq)]
53pub struct TreeDiffStructured {
54    /// Tree diff metadata (paths, modes, oids, status).
55    pub entry: DiffEntry,
56    /// Structured hunks and mode metadata.
57    pub structured: StructuredFileDiff,
58    /// `true` when either side looks like a binary blob (NUL in the first 8 KiB).
59    pub binary: bool,
60    /// Worktree/index side bytes when content is not materialized in the ODB.
61    pub patch_bytes: Option<(SideBytes, SideBytes)>,
62}
63
64/// One tree diff entry with both sides loaded for line-level diffing.
65#[derive(Debug, Clone, PartialEq, Eq)]
66pub struct TreeDiffContent {
67    /// Tree diff metadata (paths, modes, oids, status).
68    pub entry: DiffEntry,
69    /// Old-side bytes prepared for [`super::structured::structured_text_diff`].
70    pub old_bytes: SideBytes,
71    /// New-side bytes prepared for structured diffing.
72    pub new_bytes: SideBytes,
73    /// `true` when either side looks like a binary blob (NUL in the first 8 KiB).
74    pub binary: bool,
75}
76
77/// Load old/new blob bytes for each entry, using a worker pool when thresholds allow.
78///
79/// Submodule gitlinks are synthesized as `Subproject commit <oid>` lines; missing sides
80/// are empty. Order matches `entries`.
81///
82/// # Errors
83///
84/// Propagates object database read failures.
85pub fn tree_diff_contents_parallel(
86    odb: &Odb,
87    entries: Vec<DiffEntry>,
88    config: &ConfigSet,
89    thread_policy: &ThreadPolicy,
90) -> Result<Vec<TreeDiffContent>> {
91    if entries.is_empty() {
92        return Ok(Vec::new());
93    }
94    let mut par_opts = thread_policy.resolve(ParallelOp::DiffContent, config);
95    par_opts.min_items = par_opts.min_items.min(8);
96    let est_bytes = entries.len().saturating_mul(8192);
97    let rows = parallel_map_indices(odb, entries.len(), par_opts, est_bytes, |i| {
98        let entry = &entries[i];
99        let (old_bytes, old_bin) = load_old_side(odb, entry, None)?;
100        let (new_bytes, new_bin) = load_new_side(odb, entry, None)?;
101        Ok::<TreeDiffContent, Error>(TreeDiffContent {
102            entry: entry.clone(),
103            old_bytes,
104            new_bytes,
105            binary: old_bin || new_bin,
106        })
107    })?;
108    Ok(rows)
109}
110
111/// Per-file line counts for stat modes (loads blobs, no hunk allocation).
112///
113/// # Errors
114///
115/// Propagates object database read failures.
116pub fn tree_diff_stat_parallel(
117    odb: &Odb,
118    entries: Vec<DiffEntry>,
119    config: &ConfigSet,
120    thread_policy: &ThreadPolicy,
121    diff_options: DiffOptions,
122) -> Result<Vec<(DiffEntry, bool, Option<LineStat>)>> {
123    if entries.is_empty() {
124        return Ok(Vec::new());
125    }
126    let par_opts = per_entry_parallel_options(thread_policy, config, entries.len());
127    let line_diff_opts = diff_options;
128    let blob_counter = line_diff_opts.blob_read_counter.clone();
129    let store = odb.pack_store_for(odb.objects_dir());
130    let rows = try_par_map_with_local(
131        &entries,
132        par_opts,
133        0,
134        || PackReadContextGuard::install(Arc::clone(&store)),
135        move |_ctx, entry| {
136            let blob_reads = blob_counter.as_deref();
137            if !entry.old_oid.is_zero()
138                && entry.old_oid == entry.new_oid
139                && entry.old_mode == entry.new_mode
140            {
141                let (_, old_bin) = load_old_side(odb, entry, blob_reads)?;
142                let binary = old_bin;
143                return Ok((
144                    entry.clone(),
145                    binary,
146                    if binary {
147                        None
148                    } else {
149                        Some(LineStat::default())
150                    },
151                ));
152            }
153            let (old_bytes, old_bin) = load_old_side(odb, entry, blob_reads)?;
154            let (new_bytes, new_bin) = load_new_side(odb, entry, blob_reads)?;
155            let binary = old_bin || new_bin;
156            let stat = if binary {
157                None
158            } else {
159                Some(line_change_counts(
160                    &String::from_utf8_lossy(&old_bytes.data),
161                    &String::from_utf8_lossy(&new_bytes.data),
162                    &line_diff_opts,
163                ))
164            };
165            Ok((entry.clone(), binary, stat))
166        },
167    );
168    match rows {
169        Ok(rows) => Ok(rows),
170        Err(ParallelHashError::Task(err)) => Err(err),
171    }
172}
173
174/// Load sides and compute a structured line diff for one tree diff entry.
175///
176/// # Errors
177///
178/// Propagates object database read failures.
179pub fn tree_diff_structured_entry(
180    odb: &Odb,
181    entry: &DiffEntry,
182    diff_options: &DiffOptions,
183) -> Result<TreeDiffStructured> {
184    let blob_reads = diff_options.blob_read_counter.as_deref();
185    if !entry.old_oid.is_zero()
186        && entry.old_oid == entry.new_oid
187        && entry.old_mode == entry.new_mode
188    {
189        let (old_bytes, old_bin) = load_old_side(odb, entry, blob_reads)?;
190        return Ok(TreeDiffStructured {
191            entry: entry.clone(),
192            structured: StructuredFileDiff {
193                old_mode: entry.old_mode.octal_if_present(),
194                new_mode: entry.new_mode.octal_if_present(),
195                old_encoding_lossy: old_bytes.encoding_lossy,
196                new_encoding_lossy: old_bytes.encoding_lossy,
197                hunks: Vec::new(),
198            },
199            binary: old_bin,
200            patch_bytes: None,
201        });
202    }
203    let (old_bytes, old_bin) = load_old_side(odb, entry, blob_reads)?;
204    let (new_bytes, new_bin) = load_new_side(odb, entry, blob_reads)?;
205    let binary = old_bin || new_bin;
206    let structured = if binary {
207        StructuredFileDiff {
208            old_mode: entry.old_mode.octal_if_present(),
209            new_mode: entry.new_mode.octal_if_present(),
210            old_encoding_lossy: old_bytes.encoding_lossy,
211            new_encoding_lossy: new_bytes.encoding_lossy,
212            hunks: Vec::new(),
213        }
214    } else {
215        structured_file_diff(
216            entry.old_mode,
217            entry.new_mode,
218            &old_bytes,
219            &new_bytes,
220            diff_options.clone(),
221        )
222    };
223    Ok(TreeDiffStructured {
224        entry: entry.clone(),
225        structured,
226        binary,
227        patch_bytes: None,
228    })
229}
230
231/// Visit structured tree diffs in path order without retaining every file's hunks at once.
232///
233/// When thread policy allows, workers compute up to about `2 × threads` entries ahead while
234/// the callback consumes results strictly in order.
235///
236/// # Errors
237///
238/// Propagates object database read failures and callback errors.
239pub fn for_each_tree_diff_structured(
240    odb: &Odb,
241    entries: &[DiffEntry],
242    diff_options: DiffOptions,
243    config: &ConfigSet,
244    thread_policy: &ThreadPolicy,
245    mut visit: impl FnMut(TreeDiffStructured) -> Result<()>,
246) -> Result<()> {
247    if entries.is_empty() {
248        return Ok(());
249    }
250    let diff_options = diff_options.cap_word_level_inline_for_changed_files(entries.len());
251    let par_opts = per_entry_parallel_options(thread_policy, config, entries.len());
252    let n_workers = par_opts.threads.get();
253    if n_workers <= 1 || entries.len() < 2 {
254        for entry in entries {
255            visit(tree_diff_structured_entry(odb, entry, &diff_options)?)?;
256        }
257        return Ok(());
258    }
259    let line_diff_opts = diff_options;
260    let store = odb.pack_store_for(odb.objects_dir());
261    run_ordered_batches(
262        entries,
263        1,
264        n_workers,
265        || PackReadContextGuard::install(Arc::clone(&store)),
266        move |_ctx, entry| tree_diff_structured_entry(odb, entry, &line_diff_opts),
267        move |_batch, rows| {
268            for row in rows {
269                visit(row)?;
270            }
271            Ok(())
272        },
273    )
274}
275
276/// Load sides and compute structured line diffs for each entry in parallel.
277///
278/// # Errors
279///
280/// Propagates object database read failures.
281pub fn tree_diff_structured_parallel(
282    odb: &Odb,
283    entries: Vec<DiffEntry>,
284    config: &ConfigSet,
285    thread_policy: &ThreadPolicy,
286    diff_options: DiffOptions,
287) -> Result<Vec<TreeDiffStructured>> {
288    if entries.is_empty() {
289        return Ok(Vec::new());
290    }
291    let diff_options = diff_options.cap_word_level_inline_for_changed_files(entries.len());
292    let par_opts = per_entry_parallel_options(thread_policy, config, entries.len());
293    let line_diff_opts = diff_options;
294    let store = odb.pack_store_for(odb.objects_dir());
295    let rows = try_par_map_with_local(
296        &entries,
297        par_opts,
298        0,
299        || PackReadContextGuard::install(Arc::clone(&store)),
300        move |_ctx, entry| tree_diff_structured_entry(odb, entry, &line_diff_opts),
301    );
302    match rows {
303        Ok(rows) => Ok(rows),
304        Err(ParallelHashError::Task(err)) => Err(err),
305    }
306}
307
308fn load_old_side(
309    odb: &Odb,
310    e: &DiffEntry,
311    blob_reads: Option<&AtomicUsize>,
312) -> Result<(SideBytes, bool)> {
313    if e.old_mode == FileMode::GITLINK {
314        return Ok((
315            SideBytes::from_bytes(gitlink_line(&e.old_oid).into_bytes()),
316            false,
317        ));
318    }
319    if e.old_mode == FileMode::ABSENT {
320        return Ok((SideBytes::from_bytes(Vec::new()), false));
321    }
322    blob_bytes(odb, &e.old_oid, blob_reads)
323}
324
325fn load_new_side(
326    odb: &Odb,
327    e: &DiffEntry,
328    blob_reads: Option<&AtomicUsize>,
329) -> Result<(SideBytes, bool)> {
330    if e.new_mode == FileMode::GITLINK {
331        return Ok((
332            SideBytes::from_bytes(gitlink_line(&e.new_oid).into_bytes()),
333            false,
334        ));
335    }
336    if e.new_mode == FileMode::ABSENT {
337        return Ok((SideBytes::from_bytes(Vec::new()), false));
338    }
339    blob_bytes(odb, &e.new_oid, blob_reads)
340}
341
342fn gitlink_line(oid: &ObjectId) -> String {
343    if *oid == ObjectId::zero() {
344        return String::new();
345    }
346    format!("Subproject commit {}\n", oid.to_hex())
347}
348
349fn blob_bytes(
350    odb: &Odb,
351    oid: &ObjectId,
352    blob_reads: Option<&AtomicUsize>,
353) -> Result<(SideBytes, bool)> {
354    if *oid == ObjectId::zero() {
355        return Ok((SideBytes::from_bytes(Vec::new()), false));
356    }
357    note_blob_read(blob_reads);
358    let data = odb.read(oid)?.data;
359    Ok(bytes_for_diff(data))
360}
361
362fn bytes_for_diff(data: Vec<u8>) -> (SideBytes, bool) {
363    if data.iter().take(8000).any(|&b| b == 0) {
364        (SideBytes::from_bytes(Vec::new()), true)
365    } else {
366        (SideBytes::from_bytes(data), false)
367    }
368}
369
370#[cfg(test)]
371mod tests {
372    use super::*;
373    use crate::config::ConfigSet;
374    use crate::diff::{diff_trees, DiffOptions, DiffStatus};
375    use crate::objects::{serialize_tree, ObjectKind, TreeEntry};
376    use crate::parallel::ThreadPolicy;
377    use crate::repo::{InitOptions, Repository};
378
379    fn init_repo() -> (tempfile::TempDir, Repository) {
380        let dir = tempfile::tempdir().expect("tempdir");
381        let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
382        (dir, repo)
383    }
384
385    #[test]
386    fn tree_diff_contents_parallel_matches_serial_load() {
387        let (_dir, repo) = init_repo();
388        let odb = repo.odb();
389        let a = odb.write(ObjectKind::Blob, b"alpha\n").expect("write a");
390        let b = odb.write(ObjectKind::Blob, b"beta\n").expect("write b");
391        let tree_a = odb
392            .write(
393                ObjectKind::Tree,
394                &serialize_tree(&[TreeEntry {
395                    mode: 0o100644,
396                    name: b"file-a".to_vec(),
397                    oid: a,
398                }]),
399            )
400            .expect("tree a");
401        let tree_b = odb
402            .write(
403                ObjectKind::Tree,
404                &serialize_tree(&[TreeEntry {
405                    mode: 0o100644,
406                    name: b"file-b".to_vec(),
407                    oid: b,
408                }]),
409            )
410            .expect("tree b");
411        let raw = diff_trees(odb, Some(&tree_a), Some(&tree_b), "").expect("diff");
412        assert_eq!(raw.len(), 2);
413        let config = repo.config().expect("config");
414        let serial = tree_diff_contents_parallel(
415            odb,
416            raw.clone(),
417            &config,
418            &ThreadPolicy::with_threads_override(Some(1)),
419        )
420        .expect("serial load");
421        let parallel = tree_diff_contents_parallel(
422            odb,
423            raw,
424            &config,
425            &ThreadPolicy::with_threads_override(Some(4)),
426        )
427        .expect("parallel load");
428        assert_eq!(serial, parallel);
429        assert!(serial
430            .iter()
431            .any(|c| c.entry.status == DiffStatus::Added || c.entry.status == DiffStatus::Deleted));
432    }
433
434    #[test]
435    fn name_only_does_not_read_blobs_for_tree_to_tree() {
436        use crate::config::ConfigSet;
437        use crate::diff::{build_file_diffs, DiffDetail, DiffOptions};
438
439        let (_dir, repo) = init_repo();
440        let odb = repo.odb();
441        let a = odb.write(ObjectKind::Blob, b"alpha\n").expect("write a");
442        let b = odb.write(ObjectKind::Blob, b"beta\n").expect("write b");
443        let old = odb
444            .write(
445                ObjectKind::Tree,
446                &serialize_tree(&[TreeEntry {
447                    mode: 0o100644,
448                    name: b"f.txt".to_vec(),
449                    oid: a,
450                }]),
451            )
452            .expect("old tree");
453        let new = odb
454            .write(
455                ObjectKind::Tree,
456                &serialize_tree(&[TreeEntry {
457                    mode: 0o100644,
458                    name: b"f.txt".to_vec(),
459                    oid: b,
460                }]),
461            )
462            .expect("new tree");
463        let entries = diff_trees(odb, Some(&old), Some(&new), "").expect("entries");
464        let metrics = odb.hot_path_metrics();
465        metrics.set_blob_counting(true);
466        metrics.reset_blob_content_reads();
467        let config = ConfigSet::new();
468        let _ = build_file_diffs(
469            odb,
470            entries,
471            DiffDetail::NameOnly,
472            DiffOptions::default(),
473            &config,
474            &ThreadPolicy::default(),
475        )
476        .expect("name-only");
477        metrics.set_blob_counting(false);
478        assert_eq!(metrics.blob_content_reads(), 0);
479    }
480
481    #[test]
482    fn single_entry_runs_serially_without_consulting_the_policy() {
483        let config = ConfigSet::new();
484        let policy = ThreadPolicy::with_threads_override(Some(4));
485        for count in [0, 1] {
486            let opts = per_entry_parallel_options(&policy, &config, count);
487            assert_eq!(opts.threads, NonZeroUsize::MIN);
488        }
489        let opts = per_entry_parallel_options(&policy, &config, 2);
490        assert_eq!(opts.min_items, 0);
491        assert_eq!(opts.min_total_bytes, None);
492        assert_eq!(
493            opts.threads,
494            policy.resolve(ParallelOp::DiffContent, &config).threads
495        );
496    }
497
498    #[test]
499    fn single_entry_structured_diff_matches_threaded_policy() {
500        let (_dir, repo) = init_repo();
501        let odb = repo.odb();
502        let a = odb.write(ObjectKind::Blob, b"one\ntwo\n").expect("write a");
503        let b = odb
504            .write(ObjectKind::Blob, b"one\nthree\n")
505            .expect("write b");
506        let tree = |oid| {
507            odb.write(
508                ObjectKind::Tree,
509                &serialize_tree(&[TreeEntry {
510                    mode: 0o100644,
511                    name: b"f.txt".to_vec(),
512                    oid,
513                }]),
514            )
515            .expect("tree")
516        };
517        let (old, new) = (tree(a), tree(b));
518        let entries = diff_trees(odb, Some(&old), Some(&new), "").expect("entries");
519        assert_eq!(entries.len(), 1);
520        let config = repo.config().expect("config");
521        let run = |threads| {
522            tree_diff_structured_parallel(
523                odb,
524                entries.clone(),
525                &config,
526                &ThreadPolicy::with_threads_override(Some(threads)),
527                DiffOptions::default(),
528            )
529            .expect("structured diff")
530        };
531        let serial = run(1);
532        assert_eq!(serial, run(4));
533        assert_eq!(serial[0].structured.hunks.len(), 1);
534    }
535}
grit_lib/diff/tree_content.rs

grit_lib/diff/
tree_content.rs
