clankerdiff_core/
parser.rs

1//! Normalization adapters for Git patches and porcelain status output.
2
3use crate::{
4    DiffDocument, DiffError, DiffScope, DiffSide, FileDiff, FileStatus, Hunk, ModeChange,
5    PatchLine, PatchLineKind, RepoPath, RepoPathError, StageState,
6};
7use diffy::{
8    Line, Patch,
9    patch_set::{FileMode, FileOperation, FilePatch, ParseOptions, PatchKind, PatchSet},
10};
11use std::collections::HashMap;
12
13const MAX_TRACKED_PATCH_BYTES: u64 = 8 * 1024 * 1024;
14const MAX_TRACKED_PATCH_LINES: usize = 20_000;
15
16/// One entry from `git status --porcelain=v1 -z`.
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub struct GitStatusEntry {
19    /// Current repository-relative path.
20    pub path: RepoPath,
21    /// Previous path for a rename or copy.
22    pub old_path: Option<RepoPath>,
23    /// Status represented by the most relevant porcelain column.
24    pub status: FileStatus,
25    /// Whether the change is in the index, worktree, or both.
26    pub staged: StageState,
27    /// Raw index status character (`X`).
28    pub index: Option<char>,
29    /// Raw worktree status character (`Y`).
30    pub worktree: Option<char>,
31}
32
33/// Contents for an untracked file, supplied by a native Git host.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub struct UntrackedFile {
36    /// Repository-relative path.
37    pub path: RepoPath,
38    pub contents: Vec<u8>,
39    pub omitted_bytes: Option<u64>,
40}
41
42/// Parses and normalizes a multi-file Git diff.
43///
44/// # Errors
45///
46/// Returns an error when the patch is malformed, contains unsupported path
47/// encoding, or contains an invalid repository-relative path.
48pub fn parse_git_diff(bytes: &[u8]) -> Result<Vec<FileDiff>, DiffError> {
49    parse_git_diff_with_path_mapper(bytes, |path| RepoPath::new(path))
50}
51
52pub fn parse_git_diff_with_path_mapper<T>(
53    bytes: &[u8],
54    mut map_path: T,
55) -> Result<Vec<FileDiff>, DiffError>
56where
57    T: FnMut(&str) -> Result<RepoPath, RepoPathError>,
58{
59    if bytes.iter().all(u8::is_ascii_whitespace) {
60        return Ok(Vec::new());
61    }
62    PatchSet::parse_bytes(bytes, ParseOptions::gitdiff())
63        .map(|patch| {
64            normalize_patch(
65                &patch.map_err(|source| DiffError::Parse { source })?,
66                &mut map_path,
67            )
68        })
69        .collect()
70}
71
72fn normalize_patch<T>(patch: &FilePatch<'_, [u8]>, map_path: &mut T) -> Result<FileDiff, DiffError>
73where
74    T: FnMut(&str) -> Result<RepoPath, RepoPathError>,
75{
76    let (previous, current, status) = normalize_operation(patch.operation(), map_path)?;
77    let (hunks, omitted_bytes) = match patch.patch() {
78        PatchKind::Text(text) => {
79            let (lines, bytes) = text_patch_size(text);
80            if lines > MAX_TRACKED_PATCH_LINES || bytes > MAX_TRACKED_PATCH_BYTES {
81                (Vec::new(), Some(bytes))
82            } else {
83                (normalize_hunks(text)?, None)
84            }
85        }
86        PatchKind::Binary(_) => (Vec::new(), None),
87    };
88    let no_newline_at_end = hunks
89        .iter()
90        .flat_map(|hunk| &hunk.lines)
91        .any(|line| line.no_newline);
92    Ok(FileDiff {
93        old_path: previous,
94        path: current,
95        status,
96        staged: StageState::Unstaged,
97        hunks,
98        binary: patch.patch().is_binary(),
99        mode: normalize_mode(patch),
100        no_newline_at_end,
101        omitted_bytes,
102        old_source: FileDiff::uncaptured_source(status, DiffSide::Old),
103        new_source: FileDiff::uncaptured_source(status, DiffSide::New),
104    })
105}
106
107fn text_patch_size(text: &Patch<'_, [u8]>) -> (usize, u64) {
108    text.hunks().iter().flat_map(diffy::Hunk::lines).fold(
109        (0_usize, 0_u64),
110        |(lines, bytes), line| {
111            let bytes_in_line = match line {
112                Line::Context(bytes) | Line::Delete(bytes) | Line::Insert(bytes) => bytes.len(),
113            };
114            (
115                lines.saturating_add(1),
116                bytes.saturating_add(u64::try_from(bytes_in_line).unwrap_or(u64::MAX)),
117            )
118        },
119    )
120}
121
122fn normalize_operation<T>(
123    operation: &FileOperation<'_, [u8]>,
124    map_path: &mut T,
125) -> Result<(Option<RepoPath>, RepoPath, FileStatus), DiffError>
126where
127    T: FnMut(&str) -> Result<RepoPath, RepoPathError>,
128{
129    let mut decode_path =
130        |bytes: &[u8], prefix: Option<&[u8]>| decode_path_with_mapper(bytes, prefix, map_path);
131    Ok(match operation {
132        FileOperation::Create(raw) => (None, decode_path(raw, Some(b"b/"))?, FileStatus::Added),
133        FileOperation::Delete(raw) => {
134            let path = decode_path(raw, Some(b"a/"))?;
135            (Some(path.clone()), path, FileStatus::Deleted)
136        }
137        FileOperation::Modify { original, modified } => {
138            let before = decode_path(original, Some(b"a/"))?;
139            let after = decode_path(modified, Some(b"b/"))?;
140            let status = if before == after {
141                FileStatus::Modified
142            } else {
143                FileStatus::Renamed
144            };
145            (Some(before), after, status)
146        }
147        FileOperation::Rename { from, to } => (
148            Some(decode_path(from, None)?),
149            decode_path(to, None)?,
150            FileStatus::Renamed,
151        ),
152        FileOperation::Copy { from, to } => (
153            Some(decode_path(from, None)?),
154            decode_path(to, None)?,
155            FileStatus::Copied,
156        ),
157    })
158}
159
160fn normalize_hunks(text: &Patch<'_, [u8]>) -> Result<Vec<Hunk>, DiffError> {
161    text.hunks()
162        .iter()
163        .map(|hunk| {
164            let old = hunk.old_range();
165            let new = hunk.new_range();
166            let mut old_line = old.start();
167            let mut new_line = new.start();
168            let lines = hunk
169                .lines()
170                .iter()
171                .map(|line| {
172                    let (kind, bytes) = match line {
173                        Line::Context(bytes) => (PatchLineKind::Context, *bytes),
174                        Line::Delete(bytes) => (PatchLineKind::Removed, *bytes),
175                        Line::Insert(bytes) => (PatchLineKind::Added, *bytes),
176                    };
177                    let old_line_no = (kind != PatchLineKind::Added).then(|| {
178                        let number = old_line;
179                        old_line += 1;
180                        number
181                    });
182                    let new_line_no = (kind != PatchLineKind::Removed).then(|| {
183                        let number = new_line;
184                        new_line += 1;
185                        number
186                    });
187                    let no_newline = !bytes.ends_with(b"\n");
188                    let text = bytes.strip_suffix(b"\n").unwrap_or(bytes);
189                    let text = std::str::from_utf8(text.strip_suffix(b"\r").unwrap_or(text))?;
190                    Ok(PatchLine {
191                        kind,
192                        text: text.into(),
193                        old_line_no,
194                        new_line_no,
195                        no_newline,
196                    })
197                })
198                .collect::<Result<Vec<_>, DiffError>>()?;
199            let function_context = hunk
200                .function_context()
201                .map(|bytes| {
202                    std::str::from_utf8(bytes)
203                        .map(|context| context.trim_end_matches(['\r', '\n']).to_owned())
204                })
205                .transpose()?;
206            let suffix = function_context
207                .as_deref()
208                .map_or_else(String::new, |context| format!(" {context}"));
209            Ok(Hunk {
210                header: format!("@@ -{old} +{new} @@{suffix}"),
211                function_context,
212                old_start: old.start(),
213                old_count: old.len(),
214                new_start: new.start(),
215                new_count: new.len(),
216                lines,
217            })
218        })
219        .collect()
220}
221
222fn normalize_mode(patch: &FilePatch<'_, [u8]>) -> Option<ModeChange> {
223    let old = patch.old_mode().copied().map(mode_string);
224    let new = patch.new_mode().copied().map(mode_string);
225    (old != new && (old.is_some() || new.is_some())).then_some(ModeChange { old, new })
226}
227
228fn decode_path(bytes: &[u8], expected_prefix: Option<&[u8]>) -> Result<RepoPath, DiffError> {
229    decode_path_with_mapper(bytes, expected_prefix, &mut |path| RepoPath::new(path))
230}
231
232fn decode_path_with_mapper<T>(
233    bytes: &[u8],
234    expected_prefix: Option<&[u8]>,
235    map_path: &mut T,
236) -> Result<RepoPath, DiffError>
237where
238    T: FnMut(&str) -> Result<RepoPath, RepoPathError>,
239{
240    let bytes = expected_prefix
241        .and_then(|prefix| bytes.strip_prefix(prefix))
242        .unwrap_or(bytes);
243    let path = std::str::from_utf8(bytes).map_err(DiffError::UnsupportedPathEncoding)?;
244    map_path(path).map_err(DiffError::InvalidPath)
245}
246
247fn mode_string(mode: FileMode) -> String {
248    match mode {
249        FileMode::Regular => "100644",
250        FileMode::Executable => "100755",
251        FileMode::Symlink => "120000",
252        FileMode::Gitlink => "160000",
253    }
254    .to_owned()
255}
256
257/// Parses `git status --porcelain=v1 -z` output.
258///
259/// # Errors
260///
261/// Returns an error for malformed records, unsupported path encoding, or
262/// invalid repository-relative paths.
263pub fn parse_porcelain_v1_z(bytes: &[u8]) -> Result<Vec<GitStatusEntry>, DiffError> {
264    let mut fields = bytes.split(|byte| *byte == 0);
265    let mut entries = Vec::new();
266    while let Some(field) = fields.next() {
267        if field.is_empty() {
268            continue;
269        }
270        if field.len() < 3 || field[2] != b' ' {
271            return Err(DiffError::InvalidPorcelainEntry);
272        }
273        let index_column = char::from(field[0]);
274        let worktree_column = char::from(field[1]);
275        let path = decode_path(&field[3..], None)?;
276        let old_path = if matches!(index_column, 'R' | 'C') || matches!(worktree_column, 'R' | 'C')
277        {
278            let previous = fields.next().filter(|field| !field.is_empty());
279            Some(decode_path(
280                previous.ok_or(DiffError::InvalidPorcelainEntry)?,
281                None,
282            )?)
283        } else {
284            None
285        };
286        let index = status_column(index_column);
287        let worktree = status_column(worktree_column);
288        let staged = if index.is_none() || index_column == '?' {
289            StageState::Unstaged
290        } else if worktree.is_some() {
291            StageState::PartiallyStaged
292        } else {
293            StageState::Staged
294        };
295        let relevant = if worktree_column == ' ' {
296            index_column
297        } else {
298            worktree_column
299        };
300        entries.push(GitStatusEntry {
301            path,
302            old_path,
303            status: file_status(relevant),
304            staged,
305            index,
306            worktree,
307        });
308    }
309    Ok(entries)
310}
311
312fn status_column(value: char) -> Option<char> {
313    (!matches!(value, ' ' | '!')).then_some(value)
314}
315
316const fn file_status(value: char) -> FileStatus {
317    match value {
318        'A' => FileStatus::Added,
319        'D' => FileStatus::Deleted,
320        'R' => FileStatus::Renamed,
321        'C' => FileStatus::Copied,
322        '?' => FileStatus::Untracked,
323        _ => FileStatus::Modified,
324    }
325}
326
327impl DiffDocument {
328    /// Builds a document from Git patch and porcelain status output.
329    ///
330    /// # Errors
331    ///
332    /// Returns an error when either Git output cannot be parsed or normalized.
333    pub fn from_git_outputs(
334        repo_root: impl Into<String>,
335        diff: &[u8],
336        porcelain: &[u8],
337        scope: DiffScope,
338    ) -> Result<Self, DiffError> {
339        Self::from_git_outputs_with_untracked(repo_root, diff, porcelain, scope, &[])
340    }
341
342    /// Builds a document from Git outputs plus separately loaded untracked files.
343    ///
344    /// # Errors
345    ///
346    /// Returns an error when Git output or untracked file data cannot be parsed
347    /// or normalized.
348    pub fn from_git_outputs_with_untracked(
349        repo_root: impl Into<String>,
350        diff: &[u8],
351        porcelain: &[u8],
352        scope: DiffScope,
353        untracked: &[UntrackedFile],
354    ) -> Result<Self, DiffError> {
355        let statuses = parse_porcelain_v1_z(porcelain)?;
356        let mut files = parse_git_diff(diff)?;
357        let mut statuses_by_path = HashMap::with_capacity(statuses.len().saturating_mul(2));
358        for status in &statuses {
359            statuses_by_path
360                .entry(status.path.clone())
361                .or_insert(status);
362            if let Some(old_path) = &status.old_path {
363                statuses_by_path.entry(old_path.clone()).or_insert(status);
364            }
365        }
366        let fallback_stage = match scope {
367            DiffScope::Staged => StageState::Staged,
368            DiffScope::Unstaged | DiffScope::Both => StageState::Unstaged,
369        };
370        for file in &mut files {
371            let status = statuses_by_path.get(&file.path).copied().or_else(|| {
372                file.old_path
373                    .as_ref()
374                    .and_then(|path| statuses_by_path.get(path).copied())
375            });
376            match status {
377                Some(status) => {
378                    file.staged = status.staged;
379                    if file.status == FileStatus::Modified {
380                        file.status = status.status;
381                    }
382                }
383                None => file.staged = fallback_stage,
384            }
385        }
386
387        for untracked_file in untracked {
388            if files.iter().any(|file| file.path == untracked_file.path) {
389                continue;
390            }
391            files.push(untracked_diff(untracked_file)?);
392        }
393
394        Ok(Self {
395            repo_root: repo_root.into(),
396            files,
397        })
398    }
399}
400
401fn untracked_diff(file: &UntrackedFile) -> Result<FileDiff, DiffError> {
402    let mut diff = match (file.omitted_bytes, std::str::from_utf8(&file.contents)) {
403        (None, Ok(text)) if !text.contains('\0') => {
404            FileDiff::from_texts(file.path.clone(), "", text)?
405        }
406        _ => FileDiff {
407            old_path: None,
408            path: file.path.clone(),
409            status: FileStatus::Untracked,
410            staged: StageState::Unstaged,
411            hunks: Vec::new(),
412            binary: true,
413            mode: None,
414            no_newline_at_end: false,
415            omitted_bytes: file.omitted_bytes,
416            old_source: FileDiff::uncaptured_source(FileStatus::Untracked, DiffSide::Old),
417            new_source: FileDiff::uncaptured_source(FileStatus::Untracked, DiffSide::New),
418        },
419    };
420    diff.status = FileStatus::Untracked;
421    diff.staged = StageState::Unstaged;
422    Ok(diff)
423}
424
425#[cfg(test)]
426mod tests {
427    use super::*;
428
429    #[test]
430    fn parses_text_rename_modes_and_no_newline() {
431        let patch = b"diff --git a/old.rs b/new.rs\nsimilarity index 80%\nrename from old.rs\nrename to new.rs\nold mode 100644\nnew mode 100755\n--- a/old.rs\n+++ b/new.rs\n@@ -1 +1 @@ function\n-old\n+new\n\\ No newline at end of file\n";
432        let files = parse_git_diff(patch).unwrap();
433        assert_eq!(files[0].status, FileStatus::Renamed);
434        assert_eq!(files[0].path.as_str(), "new.rs");
435        assert_eq!(
436            files[0].mode.as_ref().unwrap().new.as_deref(),
437            Some("100755")
438        );
439        assert!(files[0].hunks[0].lines[1].no_newline);
440        assert!(files[0].no_newline_at_end);
441        assert_eq!(
442            files[0].hunks[0].function_context.as_deref(),
443            Some("function")
444        );
445        assert_eq!(files[0].hunks[0].header, "@@ -1 +1 @@ function");
446    }
447
448    #[test]
449    fn oversized_tracked_patches_are_omitted() {
450        let mut patch = String::from(
451            "diff --git a/generated.c b/generated.c\n--- a/generated.c\n+++ b/generated.c\n@@ -1,20001 +0,0 @@\n",
452        );
453        for _ in 0..=MAX_TRACKED_PATCH_LINES {
454            patch.push_str("-generated line\n");
455        }
456
457        let files = parse_git_diff(patch.as_bytes()).unwrap();
458        assert_eq!(files.len(), 1);
459        assert!(files[0].hunks.is_empty());
460        assert!(files[0].omitted_bytes.is_some());
461        assert!(!files[0].binary);
462    }
463
464    #[test]
465    fn numbers_lines_on_the_sides_they_belong_to() {
466        let patch = b"diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1,2 +1,2 @@\n keep\n-old\n+new\n";
467        let lines = &parse_git_diff(patch).unwrap()[0].hunks[0].lines;
468        assert_eq!(
469            (lines[0].old_line_no, lines[0].new_line_no),
470            (Some(1), Some(1))
471        );
472        assert_eq!(
473            (lines[1].old_line_no, lines[1].new_line_no),
474            (Some(2), None)
475        );
476        assert_eq!(
477            (lines[2].old_line_no, lines[2].new_line_no),
478            (None, Some(2))
479        );
480    }
481
482    #[test]
483    fn parses_nul_status_with_spaces_and_rename() {
484        let status = b" M src/a file.rs\0R  new.rs\0old.rs\0?? unicode-\xc3\xa9.rs\0";
485        let entries = parse_porcelain_v1_z(status).unwrap();
486        assert_eq!(entries.len(), 3);
487        assert_eq!(entries[0].staged, StageState::Unstaged);
488        assert_eq!(entries[1].staged, StageState::Staged);
489        assert_eq!(entries[1].old_path.as_ref().unwrap().as_str(), "old.rs");
490        assert_eq!(entries[2].status, FileStatus::Untracked);
491        assert_eq!(entries[2].staged, StageState::Unstaged);
492    }
493
494    #[test]
495    fn parses_copy_binary_mode_only_and_quoted_paths() {
496        let patch = b"diff --git a/original.rs b/copied.rs\nsimilarity index 100%\ncopy from original.rs\ncopy to copied.rs\ndiff --git \"a/tab\\tname.bin\" \"b/tab\\tname.bin\"\nindex 1234567..abcdef0 100644\nBinary files \"a/tab\\tname.bin\" and \"b/tab\\tname.bin\" differ\ndiff --git a/script.sh b/script.sh\nold mode 100644\nnew mode 100755\n";
497        let files = parse_git_diff(patch).unwrap();
498        assert_eq!(files.len(), 3);
499        assert_eq!(files[0].status, FileStatus::Copied);
500        assert_eq!(files[0].old_path.as_ref().unwrap().as_str(), "original.rs");
501        assert!(files[1].binary);
502        assert_eq!(files[1].path.as_str(), "tab\tname.bin");
503        assert_eq!(
504            files[2].mode.as_ref().unwrap().old.as_deref(),
505            Some("100644")
506        );
507    }
508
509    #[test]
510    fn merges_status_and_untracked_text_and_binary() {
511        let patch = b"diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1 +1 @@\n-old\n+new\n";
512        let document = DiffDocument::from_git_outputs_with_untracked(
513            "/repo",
514            patch,
515            b"MM a.rs\0?? note.txt\0?? data.bin\0",
516            DiffScope::Both,
517            &[
518                UntrackedFile {
519                    path: RepoPath::new("note.txt").unwrap(),
520                    contents: b"hello\n".to_vec(),
521                    omitted_bytes: None,
522                },
523                UntrackedFile {
524                    path: RepoPath::new("data.bin").unwrap(),
525                    contents: b"a\0b".to_vec(),
526                    omitted_bytes: None,
527                },
528            ],
529        )
530        .unwrap();
531        assert_eq!(document.files[0].staged, StageState::PartiallyStaged);
532        assert_eq!(document.files[1].status, FileStatus::Untracked);
533        assert!(!document.files[1].binary);
534        assert!(document.files[2].binary);
535    }
536
537    #[test]
538    fn represents_omitted_untracked_content_without_diffing_it() {
539        let document = DiffDocument::from_git_outputs_with_untracked(
540            "/repo",
541            b"",
542            b"?? large.bin\0",
543            DiffScope::Unstaged,
544            &[UntrackedFile {
545                path: RepoPath::new("large.bin").unwrap(),
546                contents: Vec::new(),
547                omitted_bytes: Some(10_000_000),
548            }],
549        )
550        .unwrap();
551        assert!(document.files[0].binary);
552        assert_eq!(document.files[0].omitted_bytes, Some(10_000_000));
553        assert!(document.files[0].hunks.is_empty());
554    }
555
556    #[test]
557    fn rejects_non_utf8_path() {
558        let patch = b"diff --git a/ok b/\xff\n--- a/ok\n+++ b/\xff\n@@ -1 +1 @@\n-a\n+b\n";
559        assert!(matches!(
560            parse_git_diff(patch),
561            Err(DiffError::UnsupportedPathEncoding(_))
562        ));
563    }
564
565    #[test]
566    fn rejects_malformed_porcelain_records() {
567        assert!(matches!(
568            parse_porcelain_v1_z(b"XY\0"),
569            Err(DiffError::InvalidPorcelainEntry)
570        ));
571        assert!(matches!(
572            parse_porcelain_v1_z(b"R  renamed.rs\0"),
573            Err(DiffError::InvalidPorcelainEntry)
574        ));
575    }
576}
clankerdiff_core/parser.rs

clankerdiff_core/
parser.rs
