clankerdiff_core/
parser.rs

1//! Normalization adapters for Git patches and porcelain status output.
2
3use crate::{
4    DiffDocument, DiffError, DiffScope, DiffSide, FileDiff, FileStatus, Hunk, ModeChange,
5    PatchLine, PatchLineKind, RepoPath, StageState,
6};
7use diffy::{
8    Line, Patch,
9    patch_set::{FileMode, FileOperation, FilePatch, ParseOptions, PatchKind, PatchSet},
10};
11use std::collections::HashMap;
12
13const MAX_TRACKED_PATCH_BYTES: u64 = 8 * 1024 * 1024;
14const MAX_TRACKED_PATCH_LINES: usize = 20_000;
15
16/// One entry from `git status --porcelain=v1 -z`.
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub struct GitStatusEntry {
19    /// Current repository-relative path.
20    pub path: RepoPath,
21    /// Previous path for a rename or copy.
22    pub old_path: Option<RepoPath>,
23    /// Status represented by the most relevant porcelain column.
24    pub status: FileStatus,
25    /// Whether the change is in the index, worktree, or both.
26    pub staged: StageState,
27    /// Raw index status character (`X`).
28    pub index: Option<char>,
29    /// Raw worktree status character (`Y`).
30    pub worktree: Option<char>,
31}
32
33/// Contents for an untracked file, supplied by a native Git host.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub struct UntrackedFile {
36    /// Repository-relative path.
37    pub path: RepoPath,
38    pub contents: Vec<u8>,
39    pub omitted_bytes: Option<u64>,
40}
41
42/// Parses and normalizes a multi-file Git diff.
43///
44/// # Errors
45///
46/// Returns an error when the patch is malformed, contains unsupported path
47/// encoding, or contains an invalid repository-relative path.
48pub fn parse_git_diff(bytes: &[u8]) -> Result<Vec<FileDiff>, DiffError> {
49    if bytes.iter().all(u8::is_ascii_whitespace) {
50        return Ok(Vec::new());
51    }
52    PatchSet::parse_bytes(bytes, ParseOptions::gitdiff())
53        .map(|patch| normalize_patch(&patch.map_err(|source| DiffError::Parse { source })?))
54        .collect()
55}
56
57fn normalize_patch(patch: &FilePatch<'_, [u8]>) -> Result<FileDiff, DiffError> {
58    let (previous, current, status) = normalize_operation(patch.operation())?;
59    let (hunks, omitted_bytes) = match patch.patch() {
60        PatchKind::Text(text) => {
61            let (lines, bytes) = text_patch_size(text);
62            if lines > MAX_TRACKED_PATCH_LINES || bytes > MAX_TRACKED_PATCH_BYTES {
63                (Vec::new(), Some(bytes))
64            } else {
65                (normalize_hunks(text)?, None)
66            }
67        }
68        PatchKind::Binary(_) => (Vec::new(), None),
69    };
70    let no_newline_at_end = hunks
71        .iter()
72        .flat_map(|hunk| &hunk.lines)
73        .any(|line| line.no_newline);
74    Ok(FileDiff {
75        old_path: previous,
76        path: current,
77        status,
78        staged: StageState::Unstaged,
79        hunks,
80        binary: patch.patch().is_binary(),
81        mode: normalize_mode(patch),
82        no_newline_at_end,
83        omitted_bytes,
84        old_source: FileDiff::uncaptured_source(status, DiffSide::Old),
85        new_source: FileDiff::uncaptured_source(status, DiffSide::New),
86    })
87}
88
89fn text_patch_size(text: &Patch<'_, [u8]>) -> (usize, u64) {
90    text.hunks().iter().flat_map(diffy::Hunk::lines).fold(
91        (0_usize, 0_u64),
92        |(lines, bytes), line| {
93            let bytes_in_line = match line {
94                Line::Context(bytes) | Line::Delete(bytes) | Line::Insert(bytes) => bytes.len(),
95            };
96            (
97                lines.saturating_add(1),
98                bytes.saturating_add(u64::try_from(bytes_in_line).unwrap_or(u64::MAX)),
99            )
100        },
101    )
102}
103
104fn normalize_operation(
105    operation: &FileOperation<'_, [u8]>,
106) -> Result<(Option<RepoPath>, RepoPath, FileStatus), DiffError> {
107    Ok(match operation {
108        FileOperation::Create(raw) => (None, decode_path(raw, Some(b"b/"))?, FileStatus::Added),
109        FileOperation::Delete(raw) => {
110            let path = decode_path(raw, Some(b"a/"))?;
111            (Some(path.clone()), path, FileStatus::Deleted)
112        }
113        FileOperation::Modify { original, modified } => {
114            let before = decode_path(original, Some(b"a/"))?;
115            let after = decode_path(modified, Some(b"b/"))?;
116            let status = if before == after {
117                FileStatus::Modified
118            } else {
119                FileStatus::Renamed
120            };
121            (Some(before), after, status)
122        }
123        FileOperation::Rename { from, to } => (
124            Some(decode_path(from, None)?),
125            decode_path(to, None)?,
126            FileStatus::Renamed,
127        ),
128        FileOperation::Copy { from, to } => (
129            Some(decode_path(from, None)?),
130            decode_path(to, None)?,
131            FileStatus::Copied,
132        ),
133    })
134}
135
136fn normalize_hunks(text: &Patch<'_, [u8]>) -> Result<Vec<Hunk>, DiffError> {
137    text.hunks()
138        .iter()
139        .map(|hunk| {
140            let old = hunk.old_range();
141            let new = hunk.new_range();
142            let mut old_line = old.start();
143            let mut new_line = new.start();
144            let lines = hunk
145                .lines()
146                .iter()
147                .map(|line| {
148                    let (kind, bytes) = match line {
149                        Line::Context(bytes) => (PatchLineKind::Context, *bytes),
150                        Line::Delete(bytes) => (PatchLineKind::Removed, *bytes),
151                        Line::Insert(bytes) => (PatchLineKind::Added, *bytes),
152                    };
153                    let old_line_no = (kind != PatchLineKind::Added).then(|| {
154                        let number = old_line;
155                        old_line += 1;
156                        number
157                    });
158                    let new_line_no = (kind != PatchLineKind::Removed).then(|| {
159                        let number = new_line;
160                        new_line += 1;
161                        number
162                    });
163                    let no_newline = !bytes.ends_with(b"\n");
164                    let text = bytes.strip_suffix(b"\n").unwrap_or(bytes);
165                    let text = std::str::from_utf8(text.strip_suffix(b"\r").unwrap_or(text))?;
166                    Ok(PatchLine {
167                        kind,
168                        text: text.into(),
169                        old_line_no,
170                        new_line_no,
171                        no_newline,
172                    })
173                })
174                .collect::<Result<Vec<_>, DiffError>>()?;
175            let function_context = hunk
176                .function_context()
177                .map(|bytes| {
178                    std::str::from_utf8(bytes)
179                        .map(|context| context.trim_end_matches(['\r', '\n']).to_owned())
180                })
181                .transpose()?;
182            let suffix = function_context
183                .as_deref()
184                .map_or_else(String::new, |context| format!(" {context}"));
185            Ok(Hunk {
186                header: format!("@@ -{old} +{new} @@{suffix}"),
187                function_context,
188                old_start: old.start(),
189                old_count: old.len(),
190                new_start: new.start(),
191                new_count: new.len(),
192                lines,
193            })
194        })
195        .collect()
196}
197
198fn normalize_mode(patch: &FilePatch<'_, [u8]>) -> Option<ModeChange> {
199    let old = patch.old_mode().copied().map(mode_string);
200    let new = patch.new_mode().copied().map(mode_string);
201    (old != new && (old.is_some() || new.is_some())).then_some(ModeChange { old, new })
202}
203
204fn decode_path(bytes: &[u8], expected_prefix: Option<&[u8]>) -> Result<RepoPath, DiffError> {
205    let bytes = expected_prefix
206        .and_then(|prefix| bytes.strip_prefix(prefix))
207        .unwrap_or(bytes);
208    let path = std::str::from_utf8(bytes).map_err(DiffError::UnsupportedPathEncoding)?;
209    RepoPath::new(path).map_err(DiffError::InvalidPath)
210}
211
212fn mode_string(mode: FileMode) -> String {
213    match mode {
214        FileMode::Regular => "100644",
215        FileMode::Executable => "100755",
216        FileMode::Symlink => "120000",
217        FileMode::Gitlink => "160000",
218    }
219    .to_owned()
220}
221
222/// Parses `git status --porcelain=v1 -z` output.
223///
224/// # Errors
225///
226/// Returns an error for malformed records, unsupported path encoding, or
227/// invalid repository-relative paths.
228pub fn parse_porcelain_v1_z(bytes: &[u8]) -> Result<Vec<GitStatusEntry>, DiffError> {
229    let mut fields = bytes.split(|byte| *byte == 0);
230    let mut entries = Vec::new();
231    while let Some(field) = fields.next() {
232        if field.is_empty() {
233            continue;
234        }
235        if field.len() < 3 || field[2] != b' ' {
236            return Err(DiffError::InvalidPorcelainEntry);
237        }
238        let index_column = char::from(field[0]);
239        let worktree_column = char::from(field[1]);
240        let path = decode_path(&field[3..], None)?;
241        let old_path = if matches!(index_column, 'R' | 'C') || matches!(worktree_column, 'R' | 'C')
242        {
243            let previous = fields.next().filter(|field| !field.is_empty());
244            Some(decode_path(
245                previous.ok_or(DiffError::InvalidPorcelainEntry)?,
246                None,
247            )?)
248        } else {
249            None
250        };
251        let index = status_column(index_column);
252        let worktree = status_column(worktree_column);
253        let staged = if index.is_none() || index_column == '?' {
254            StageState::Unstaged
255        } else if worktree.is_some() {
256            StageState::PartiallyStaged
257        } else {
258            StageState::Staged
259        };
260        let relevant = if worktree_column == ' ' {
261            index_column
262        } else {
263            worktree_column
264        };
265        entries.push(GitStatusEntry {
266            path,
267            old_path,
268            status: file_status(relevant),
269            staged,
270            index,
271            worktree,
272        });
273    }
274    Ok(entries)
275}
276
277fn status_column(value: char) -> Option<char> {
278    (!matches!(value, ' ' | '!')).then_some(value)
279}
280
281const fn file_status(value: char) -> FileStatus {
282    match value {
283        'A' => FileStatus::Added,
284        'D' => FileStatus::Deleted,
285        'R' => FileStatus::Renamed,
286        'C' => FileStatus::Copied,
287        '?' => FileStatus::Untracked,
288        _ => FileStatus::Modified,
289    }
290}
291
292impl DiffDocument {
293    /// Builds a document from Git patch and porcelain status output.
294    ///
295    /// # Errors
296    ///
297    /// Returns an error when either Git output cannot be parsed or normalized.
298    pub fn from_git_outputs(
299        repo_root: impl Into<String>,
300        diff: &[u8],
301        porcelain: &[u8],
302        scope: DiffScope,
303    ) -> Result<Self, DiffError> {
304        Self::from_git_outputs_with_untracked(repo_root, diff, porcelain, scope, &[])
305    }
306
307    /// Builds a document from Git outputs plus separately loaded untracked files.
308    ///
309    /// # Errors
310    ///
311    /// Returns an error when Git output or untracked file data cannot be parsed
312    /// or normalized.
313    pub fn from_git_outputs_with_untracked(
314        repo_root: impl Into<String>,
315        diff: &[u8],
316        porcelain: &[u8],
317        scope: DiffScope,
318        untracked: &[UntrackedFile],
319    ) -> Result<Self, DiffError> {
320        let statuses = parse_porcelain_v1_z(porcelain)?;
321        let mut files = parse_git_diff(diff)?;
322        let mut statuses_by_path = HashMap::with_capacity(statuses.len().saturating_mul(2));
323        for status in &statuses {
324            statuses_by_path
325                .entry(status.path.clone())
326                .or_insert(status);
327            if let Some(old_path) = &status.old_path {
328                statuses_by_path.entry(old_path.clone()).or_insert(status);
329            }
330        }
331        let fallback_stage = match scope {
332            DiffScope::Staged => StageState::Staged,
333            DiffScope::Unstaged | DiffScope::Both => StageState::Unstaged,
334        };
335        for file in &mut files {
336            let status = statuses_by_path.get(&file.path).copied().or_else(|| {
337                file.old_path
338                    .as_ref()
339                    .and_then(|path| statuses_by_path.get(path).copied())
340            });
341            match status {
342                Some(status) => {
343                    file.staged = status.staged;
344                    if file.status == FileStatus::Modified {
345                        file.status = status.status;
346                    }
347                }
348                None => file.staged = fallback_stage,
349            }
350        }
351
352        for untracked_file in untracked {
353            if files.iter().any(|file| file.path == untracked_file.path) {
354                continue;
355            }
356            files.push(untracked_diff(untracked_file)?);
357        }
358
359        Ok(Self {
360            repo_root: repo_root.into(),
361            files,
362        })
363    }
364}
365
366fn untracked_diff(file: &UntrackedFile) -> Result<FileDiff, DiffError> {
367    let mut diff = match (file.omitted_bytes, std::str::from_utf8(&file.contents)) {
368        (None, Ok(text)) if !text.contains('\0') => {
369            FileDiff::from_texts(file.path.clone(), "", text)?
370        }
371        _ => FileDiff {
372            old_path: None,
373            path: file.path.clone(),
374            status: FileStatus::Untracked,
375            staged: StageState::Unstaged,
376            hunks: Vec::new(),
377            binary: true,
378            mode: None,
379            no_newline_at_end: false,
380            omitted_bytes: file.omitted_bytes,
381            old_source: FileDiff::uncaptured_source(FileStatus::Untracked, DiffSide::Old),
382            new_source: FileDiff::uncaptured_source(FileStatus::Untracked, DiffSide::New),
383        },
384    };
385    diff.status = FileStatus::Untracked;
386    diff.staged = StageState::Unstaged;
387    Ok(diff)
388}
389
390#[cfg(test)]
391mod tests {
392    use super::*;
393
394    #[test]
395    fn parses_text_rename_modes_and_no_newline() {
396        let patch = b"diff --git a/old.rs b/new.rs\nsimilarity index 80%\nrename from old.rs\nrename to new.rs\nold mode 100644\nnew mode 100755\n--- a/old.rs\n+++ b/new.rs\n@@ -1 +1 @@ function\n-old\n+new\n\\ No newline at end of file\n";
397        let files = parse_git_diff(patch).unwrap();
398        assert_eq!(files[0].status, FileStatus::Renamed);
399        assert_eq!(files[0].path.as_str(), "new.rs");
400        assert_eq!(
401            files[0].mode.as_ref().unwrap().new.as_deref(),
402            Some("100755")
403        );
404        assert!(files[0].hunks[0].lines[1].no_newline);
405        assert!(files[0].no_newline_at_end);
406        assert_eq!(
407            files[0].hunks[0].function_context.as_deref(),
408            Some("function")
409        );
410        assert_eq!(files[0].hunks[0].header, "@@ -1 +1 @@ function");
411    }
412
413    #[test]
414    fn oversized_tracked_patches_are_omitted() {
415        let mut patch = String::from(
416            "diff --git a/generated.c b/generated.c\n--- a/generated.c\n+++ b/generated.c\n@@ -1,20001 +0,0 @@\n",
417        );
418        for _ in 0..=MAX_TRACKED_PATCH_LINES {
419            patch.push_str("-generated line\n");
420        }
421
422        let files = parse_git_diff(patch.as_bytes()).unwrap();
423        assert_eq!(files.len(), 1);
424        assert!(files[0].hunks.is_empty());
425        assert!(files[0].omitted_bytes.is_some());
426        assert!(!files[0].binary);
427    }
428
429    #[test]
430    fn numbers_lines_on_the_sides_they_belong_to() {
431        let patch = b"diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1,2 +1,2 @@\n keep\n-old\n+new\n";
432        let lines = &parse_git_diff(patch).unwrap()[0].hunks[0].lines;
433        assert_eq!(
434            (lines[0].old_line_no, lines[0].new_line_no),
435            (Some(1), Some(1))
436        );
437        assert_eq!(
438            (lines[1].old_line_no, lines[1].new_line_no),
439            (Some(2), None)
440        );
441        assert_eq!(
442            (lines[2].old_line_no, lines[2].new_line_no),
443            (None, Some(2))
444        );
445    }
446
447    #[test]
448    fn parses_nul_status_with_spaces_and_rename() {
449        let status = b" M src/a file.rs\0R  new.rs\0old.rs\0?? unicode-\xc3\xa9.rs\0";
450        let entries = parse_porcelain_v1_z(status).unwrap();
451        assert_eq!(entries.len(), 3);
452        assert_eq!(entries[0].staged, StageState::Unstaged);
453        assert_eq!(entries[1].staged, StageState::Staged);
454        assert_eq!(entries[1].old_path.as_ref().unwrap().as_str(), "old.rs");
455        assert_eq!(entries[2].status, FileStatus::Untracked);
456        assert_eq!(entries[2].staged, StageState::Unstaged);
457    }
458
459    #[test]
460    fn parses_copy_binary_mode_only_and_quoted_paths() {
461        let patch = b"diff --git a/original.rs b/copied.rs\nsimilarity index 100%\ncopy from original.rs\ncopy to copied.rs\ndiff --git \"a/tab\\tname.bin\" \"b/tab\\tname.bin\"\nindex 1234567..abcdef0 100644\nBinary files \"a/tab\\tname.bin\" and \"b/tab\\tname.bin\" differ\ndiff --git a/script.sh b/script.sh\nold mode 100644\nnew mode 100755\n";
462        let files = parse_git_diff(patch).unwrap();
463        assert_eq!(files.len(), 3);
464        assert_eq!(files[0].status, FileStatus::Copied);
465        assert_eq!(files[0].old_path.as_ref().unwrap().as_str(), "original.rs");
466        assert!(files[1].binary);
467        assert_eq!(files[1].path.as_str(), "tab\tname.bin");
468        assert_eq!(
469            files[2].mode.as_ref().unwrap().old.as_deref(),
470            Some("100644")
471        );
472    }
473
474    #[test]
475    fn merges_status_and_untracked_text_and_binary() {
476        let patch = b"diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1 +1 @@\n-old\n+new\n";
477        let document = DiffDocument::from_git_outputs_with_untracked(
478            "/repo",
479            patch,
480            b"MM a.rs\0?? note.txt\0?? data.bin\0",
481            DiffScope::Both,
482            &[
483                UntrackedFile {
484                    path: RepoPath::new("note.txt").unwrap(),
485                    contents: b"hello\n".to_vec(),
486                    omitted_bytes: None,
487                },
488                UntrackedFile {
489                    path: RepoPath::new("data.bin").unwrap(),
490                    contents: b"a\0b".to_vec(),
491                    omitted_bytes: None,
492                },
493            ],
494        )
495        .unwrap();
496        assert_eq!(document.files[0].staged, StageState::PartiallyStaged);
497        assert_eq!(document.files[1].status, FileStatus::Untracked);
498        assert!(!document.files[1].binary);
499        assert!(document.files[2].binary);
500    }
501
502    #[test]
503    fn represents_omitted_untracked_content_without_diffing_it() {
504        let document = DiffDocument::from_git_outputs_with_untracked(
505            "/repo",
506            b"",
507            b"?? large.bin\0",
508            DiffScope::Unstaged,
509            &[UntrackedFile {
510                path: RepoPath::new("large.bin").unwrap(),
511                contents: Vec::new(),
512                omitted_bytes: Some(10_000_000),
513            }],
514        )
515        .unwrap();
516        assert!(document.files[0].binary);
517        assert_eq!(document.files[0].omitted_bytes, Some(10_000_000));
518        assert!(document.files[0].hunks.is_empty());
519    }
520
521    #[test]
522    fn rejects_non_utf8_path() {
523        let patch = b"diff --git a/ok b/\xff\n--- a/ok\n+++ b/\xff\n@@ -1 +1 @@\n-a\n+b\n";
524        assert!(matches!(
525            parse_git_diff(patch),
526            Err(DiffError::UnsupportedPathEncoding(_))
527        ));
528    }
529
530    #[test]
531    fn rejects_malformed_porcelain_records() {
532        assert!(matches!(
533            parse_porcelain_v1_z(b"XY\0"),
534            Err(DiffError::InvalidPorcelainEntry)
535        ));
536        assert!(matches!(
537            parse_porcelain_v1_z(b"R  renamed.rs\0"),
538            Err(DiffError::InvalidPorcelainEntry)
539        ));
540    }
541}
clankerdiff_core/parser.rs

clankerdiff_core/
parser.rs
