hf_hub/repository/
diff.rs

1//! Parsed representations of the Hub's raw diff format.
2//!
3//! [`crate::repository::HFRepository::get_raw_diff`] returns the raw compare payload as a
4//! string, while [`crate::repository::HFRepository::get_raw_diff_stream`] uses the parser
5//! in this module to yield one [`HFFileDiff`] per diff entry.
6//!
7//! The format is a line-oriented variant of git's raw diff output: each line
8//! encodes the old/new blob ids, the file status, whether the payload is text
9//! or binary, the destination-side file size, and the relevant paths. This
10//! module does not parse patch hunks; it focuses on the metadata needed to
11//! inspect revision-to-revision file changes efficiently.
12
13use bytes::Bytes;
14use futures::stream::{self, Stream, StreamExt};
15use thiserror::Error;
16use tokio::io::{AsyncBufReadExt, BufReader};
17use tokio_util::io::StreamReader;
18
19/// Errors produced while parsing raw diff lines into [`HFFileDiff`] values.
20///
21/// These errors surface on the stream returned by
22/// [`crate::repository::HFRepository::get_raw_diff_stream`] when the Hub returns a
23/// line that does not match the expected raw-diff shape, or when the response
24/// stream itself fails while being read.
25#[derive(Debug, Error)]
26pub enum HFDiffParseError {
27    /// The parser encountered an empty line, but every diff entry is expected
28    /// to occupy exactly one non-empty line.
29    #[error("diff line is empty")]
30    EmptyLine,
31    /// The size prefix in the raw diff line could not be parsed as a `u64`.
32    #[error("failed to parse file size from {value:?} in line {line:?}: {source}")]
33    InvalidFileSize {
34        /// The substring that was expected to contain the file size.
35        value: String,
36        /// The full raw diff line being parsed.
37        line: String,
38        /// The underlying integer parse failure.
39        source: std::num::ParseIntError,
40    },
41    /// The line was structurally different from the raw diff format this
42    /// parser expects.
43    #[error("incorrect diff line format: {line:?}")]
44    InvalidFormat {
45        /// The full raw diff line that could not be parsed.
46        line: String,
47    },
48    /// Reading the response stream failed before the next diff line could be
49    /// parsed.
50    #[error("I/O error while reading diff stream: {0}")]
51    Io(#[from] std::io::Error),
52}
53
54/// Parsed metadata for a single file entry in the Hub's raw diff output.
55///
56/// One `HFFileDiff` corresponds to one line in the payload returned by
57/// [`crate::repository::HFRepository::get_raw_diff`] or one item yielded by
58/// [`crate::repository::HFRepository::get_raw_diff_stream`]. For rename and copy
59/// entries, [`new_file_path`](Self::new_file_path) contains the destination
60/// path while [`file_path`](Self::file_path) remains the source path.
61#[derive(Debug, PartialEq, Eq, Clone)]
62pub struct HFFileDiff {
63    /// Blob id for the file before the change.
64    ///
65    /// For additions this is typically all zeroes because there is no previous
66    /// blob.
67    pub old_blob_id: String,
68    /// Blob id for the file after the change.
69    ///
70    /// For deletions this is typically all zeroes because there is no new
71    /// blob.
72    pub new_blob_id: String,
73    /// High-level git status for the change.
74    pub status: GitStatus,
75    /// Repository-relative path of the original/source file.
76    pub file_path: String,
77    /// Repository-relative destination path for rename/copy entries.
78    ///
79    /// This is `None` for additions, deletions, and in-place modifications.
80    pub new_file_path: Option<String>,
81    /// Whether the Hub marked this entry as binary.
82    ///
83    /// `true` corresponds to a `B` prefix in the raw diff line; `false`
84    /// corresponds to `T` (text).
85    pub is_binary: bool,
86    /// Destination-side file size in bytes, as reported by the raw diff line.
87    pub new_file_size: u64,
88}
89
90/// File-level status code parsed from a raw diff entry.
91///
92/// The variants correspond to the single-letter status code in git raw diff
93/// output (`A`, `C`, `D`, `M`, `R`, `T`, `U`, `X`).
94#[derive(Debug, PartialEq, Eq, Clone)]
95pub enum GitStatus {
96    /// A new file was added.
97    Addition,
98    /// The file was copied to a new path.
99    Copy,
100    /// The file was deleted.
101    Deletion,
102    /// The file contents changed in place.
103    Modification,
104    /// The file's mode/type changed without necessarily changing its path.
105    FileTypeChange,
106    /// The file was renamed.
107    Rename,
108    /// Git reported an unrecognized status code.
109    Unknown,
110    /// The entry is in an unmerged/conflicted state.
111    Unmerged,
112}
113
114impl From<char> for GitStatus {
115    fn from(value: char) -> Self {
116        match value {
117            'A' => Self::Addition,
118            'C' => Self::Copy,
119            'D' => Self::Deletion,
120            'M' => Self::Modification,
121            'R' => Self::Rename,
122            'T' => Self::FileTypeChange,
123            'U' => Self::Unmerged,
124            'X' => Self::Unknown,
125            _ => Self::Unknown,
126        }
127    }
128}
129
130/// Parse a single line of HF raw diff output into an `HFFileDiff`.
131///
132/// Format reference: <https://git-scm.com/docs/diff-format#_raw_output_format>
133fn parse_hf_diff_line(line: &str) -> Result<HFFileDiff, HFDiffParseError> {
134    let original_line = line;
135    let fmt_err = || HFDiffParseError::InvalidFormat {
136        line: original_line.to_owned(),
137    };
138    let bin_or_text = line.chars().next().ok_or(HFDiffParseError::EmptyLine)?;
139    let is_binary = bin_or_text != 'T';
140    // skip {B|T} and space
141    let line = line.get(2..).ok_or_else(&fmt_err)?;
142    let mut i = 0;
143    for char in line.chars() {
144        if !char.is_ascii_digit() {
145            break;
146        }
147        i += 1;
148    }
149    let size_str = &line[..i];
150    let new_file_size = size_str.parse().map_err(|e| HFDiffParseError::InvalidFileSize {
151        value: size_str.to_owned(),
152        line: original_line.to_owned(),
153        source: e,
154    })?;
155    // skip file size + \t
156    let line = line.get(i + 1..).ok_or_else(&fmt_err)?;
157    // skip :000000 000000 & space
158    let line = line.get(15..).ok_or_else(&fmt_err)?;
159    let old_blob_id = line.get(..40).ok_or_else(&fmt_err)?.to_owned();
160    // skip sha1;0{40}, ... & space
161    let line = line.get(44..).ok_or_else(&fmt_err)?;
162    let new_blob_id = line.get(..40).ok_or_else(&fmt_err)?.to_owned();
163    // skip sha1;0{40}, ... & space
164    let line = line.get(44..).ok_or_else(&fmt_err)?;
165    let status = line.chars().next().ok_or_else(&fmt_err)?.into();
166    let line = line.get(1..).ok_or_else(&fmt_err)?;
167    // skip optional score digits 1-3 chars & \t
168    let mut i = 0;
169    for char in line.chars() {
170        if !char.is_ascii_digit() {
171            break;
172        }
173        i += char.len_utf8();
174    }
175    let line = line.get(i + 1..).ok_or_else(&fmt_err)?;
176    let separator_is_tab = line.contains('\t');
177    // read up to next space or newline
178    let i = if matches!(status, GitStatus::Copy | GitStatus::Rename) {
179        let mut i = 0;
180        for char in line.chars() {
181            match (separator_is_tab, char) {
182                (true, '\t') => break,
183                (false, ' ') => break,
184                _ => (),
185            }
186            i += char.len_utf8();
187        }
188        i
189    } else {
190        line.len()
191    };
192    let file_path = line[..i].to_owned();
193    let line = &line[i..];
194    let new_file_path = if !line.is_empty() {
195        // skip separator
196        let line = &line[1..];
197        Some(line.to_owned())
198    } else {
199        None
200    };
201
202    Ok(HFFileDiff {
203        old_blob_id,
204        new_blob_id,
205        status,
206        file_path,
207        new_file_path,
208        is_binary,
209        new_file_size,
210    })
211}
212
213/// Stream-parse raw HF diff output line by line.
214///
215/// Takes a byte stream (e.g., from `HFRepository::get_raw_diff_stream`) and returns
216/// a stream of `HFFileDiff` items, parsing each line as it arrives without buffering
217/// the entire response.
218pub(crate) fn stream_raw_diff<S, E>(byte_stream: S) -> impl Stream<Item = Result<HFFileDiff, HFDiffParseError>> + Unpin
219where
220    S: Stream<Item = Result<Bytes, E>>,
221    E: Into<std::io::Error>,
222{
223    let mapped_stream = Box::pin(byte_stream).map(|r| r.map_err(Into::into));
224    let reader = StreamReader::new(mapped_stream);
225    let buf_reader = BufReader::new(reader);
226    let lines = buf_reader.lines();
227
228    Box::pin(stream::unfold(lines, |mut lines| async move {
229        match lines.next_line().await {
230            Ok(Some(line)) => {
231                let result = parse_hf_diff_line(&line);
232                if let Err(ref err) = result {
233                    tracing::warn!(
234                        line = %line,
235                        error = %err,
236                        "failed to parse diff line"
237                    );
238                }
239                Some((result, lines))
240            },
241            Ok(None) => None,
242            Err(e) => Some((Err(HFDiffParseError::Io(e)), lines)),
243        }
244    }))
245}
246
247#[cfg(test)]
248mod tests {
249    use futures::StreamExt;
250
251    use super::{GitStatus, HFFileDiff, parse_hf_diff_line, stream_raw_diff};
252
253    #[test]
254    fn modified_hf_diff() {
255        assert_eq!(
256            parse_hf_diff_line("T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/dispatcher.rs").unwrap(),
257            HFFileDiff {
258                old_blob_id: "97e7432a448baa9e97ec5e4f03c57b09b8e116ed".to_owned(),
259                new_blob_id: "0000000000000000000000000000000000000000".to_owned(),
260                status: GitStatus::Modification,
261                file_path: "apps/scan_orchestrator/src/dispatcher.rs".to_owned(),
262                new_file_path: None,
263                is_binary: false,
264                new_file_size: 2305,
265            }
266        );
267    }
268
269    #[test]
270    fn binary_copy_with_score() {
271        assert_eq!(
272            parse_hf_diff_line("B 421211\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... C68\tapps/scan_orchestrator/src/dispatcher.rs apps/scan_orchestrator/src/blob").unwrap(),
273            HFFileDiff {
274                old_blob_id: "97e7432a448baa9e97ec5e4f03c57b09b8e116ed".to_owned(),
275                new_blob_id: "0000000000000000000000000000000000000000".to_owned(),
276                status: GitStatus::Copy,
277                file_path: "apps/scan_orchestrator/src/dispatcher.rs".to_owned(),
278                new_file_path: Some("apps/scan_orchestrator/src/blob".to_owned()),
279                is_binary: true,
280                new_file_size: 421211,
281            }
282        );
283    }
284
285    #[test]
286    fn rename_without_score() {
287        assert_eq!(
288            parse_hf_diff_line("T 1679\t:100644 100644 f7b95e09e0573a829c338fe46e451b5609424a70... 0000000000000000000000000000000000000000... R\tapps/shared/src/scanner/file.rs apps/shared/src/scanner/file3.rs").unwrap(),
289            HFFileDiff {
290                old_blob_id: "f7b95e09e0573a829c338fe46e451b5609424a70".to_owned(),
291                new_blob_id: "0000000000000000000000000000000000000000".to_owned(),
292                status: GitStatus::Rename,
293                file_path: "apps/shared/src/scanner/file.rs".to_owned(),
294                new_file_path: Some("apps/shared/src/scanner/file3.rs".to_owned()),
295                is_binary: false,
296                new_file_size: 1679,
297            }
298        );
299    }
300
301    #[test]
302    fn unicode_and_emoji_paths() {
303        let d = parse_hf_diff_line("T 37861440\t:000000 100644 0000000000000000000000000000000000000000... 30a03d21620ebc6167e350aef9e2ac2774cf372d... A\tAI_popai/エイミ Eimi-ブルーアーカイブ Blue Archive (230270)/259889/eimi_(blue_archive).safetensors").unwrap();
304        assert_eq!(d.status, GitStatus::Addition);
305        assert_eq!(
306            d.file_path,
307            "AI_popai/エイミ Eimi-ブルーアーカイブ Blue Archive (230270)/259889/eimi_(blue_archive).safetensors"
308        );
309        assert_eq!(d.new_file_size, 37861440);
310
311        let d = parse_hf_diff_line("T 228455604\t:000000 100644 0000000000000000000000000000000000000000... 77367f06242f620081e0103c599818bfde8d4c75... D\tFaeia/💀SDXL Antler Pagan💀 (236040)/266140/SDXLAntlerPagan.safetensors").unwrap();
312        assert_eq!(d.status, GitStatus::Deletion);
313        assert!(d.file_path.contains("💀"));
314    }
315
316    #[test]
317    fn rename_with_unicode_paths() {
318        let d = parse_hf_diff_line("T 1679\t:100644 100644 f7b95e09e0573a829c338fe46e451b5609424a70... 0000000000000000000000000000000000000000... R\tエイミ/old_file.rs\tエイミ/new_file.rs").unwrap();
319        assert_eq!(d.status, GitStatus::Rename);
320        assert_eq!(d.file_path, "エイミ/old_file.rs");
321        assert_eq!(d.new_file_path, Some("エイミ/new_file.rs".to_owned()));
322    }
323
324    // A valid line used as a base for truncation tests.
325    const VALID_LINE: &str = "T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/dispatcher.rs";
326
327    fn assert_invalid_format(input: &str) {
328        match parse_hf_diff_line(input) {
329            Err(super::HFDiffParseError::InvalidFormat { .. }) => {},
330            other => panic!("expected InvalidFormat for {input:?}, got {other:?}"),
331        }
332    }
333
334    #[test]
335    fn empty_line_error() {
336        assert!(matches!(parse_hf_diff_line(""), Err(super::HFDiffParseError::EmptyLine)));
337    }
338
339    #[test]
340    fn single_char_truncated() {
341        assert_invalid_format("T");
342    }
343
344    #[test]
345    fn truncated_after_size() {
346        assert_invalid_format("T 2305");
347    }
348
349    #[test]
350    fn truncated_before_mode_pair() {
351        assert_invalid_format("T 2305\t:100644");
352    }
353
354    #[test]
355    fn truncated_before_old_blob_id() {
356        assert_invalid_format("T 2305\t:100644 100644 abcdef");
357    }
358
359    #[test]
360    fn truncated_before_new_blob_id() {
361        assert_invalid_format("T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... ");
362    }
363
364    #[test]
365    fn truncated_before_status() {
366        assert_invalid_format(
367            "T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000...",
368        );
369    }
370
371    #[test]
372    fn truncated_after_status() {
373        assert_invalid_format(
374            "T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... M",
375        );
376    }
377
378    #[test]
379    fn valid_line_parses_ok() {
380        parse_hf_diff_line(VALID_LINE).unwrap();
381    }
382
383    #[tokio::test]
384    async fn stream_modified_hf_diff() {
385        let input = b"T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/dispatcher.rs\nT 4422\t:100644 100644 c417bf5a3fbec60b22aeab13cfa4d9439155303b... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/git.rs\n";
386
387        let byte_stream = futures::stream::once(async { Ok::<_, std::io::Error>(bytes::Bytes::from(&input[..])) });
388
389        let diffs: Vec<HFFileDiff> = stream_raw_diff(byte_stream).map(|r| r.unwrap()).collect().await;
390
391        assert_eq!(diffs.len(), 2);
392        assert_eq!(diffs[0].file_path, "apps/scan_orchestrator/src/dispatcher.rs");
393        assert_eq!(diffs[0].status, GitStatus::Modification);
394        assert_eq!(diffs[0].new_file_size, 2305);
395        assert_eq!(diffs[1].file_path, "apps/scan_orchestrator/src/git.rs");
396        assert_eq!(diffs[1].status, GitStatus::Modification);
397        assert_eq!(diffs[1].new_file_size, 4422);
398    }
399
400    #[tokio::test]
401    async fn stream_mixed_statuses() {
402        let input = b"T 37861440\t:000000 100644 0000000000000000000000000000000000000000... 30a03d21620ebc6167e350aef9e2ac2774cf372d... A\tAI_popai/\xe3\x82\xa8\xe3\x82\xa4\xe3\x83\x9f Eimi (230270)/259889/eimi.safetensors\nT 228455604\t:100644 000000 77367f06242f620081e0103c599818bfde8d4c75... 0000000000000000000000000000000000000000... D\tFaeia/\xf0\x9f\x98\xa1SDXL Rage Style\xf0\x9f\x98\xa1 (234815)/264786/SDXLRageStyle.safetensors\nT 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/dispatcher.rs\nB 421211\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... C68\tapps/scan_orchestrator/src/dispatcher.rs apps/scan_orchestrator/src/blob\nT 1679\t:100644 100644 f7b95e09e0573a829c338fe46e451b5609424a70... 0000000000000000000000000000000000000000... R\tapps/shared/src/scanner/file.rs apps/shared/src/scanner/file3.rs\n";
403
404        let byte_stream = futures::stream::once(async { Ok::<_, std::io::Error>(bytes::Bytes::from(&input[..])) });
405
406        let diffs: Vec<HFFileDiff> = stream_raw_diff(byte_stream).map(|r| r.unwrap()).collect().await;
407
408        assert_eq!(diffs.len(), 5);
409        assert_eq!(diffs[0].status, GitStatus::Addition);
410        assert!(diffs[0].file_path.contains("Eimi"));
411        assert_eq!(diffs[1].status, GitStatus::Deletion);
412        assert!(diffs[1].file_path.contains("😡"));
413        assert_eq!(diffs[2].status, GitStatus::Modification);
414        assert_eq!(diffs[3].status, GitStatus::Copy);
415        assert_eq!(diffs[3].new_file_path.as_deref(), Some("apps/scan_orchestrator/src/blob"));
416        assert!(diffs[3].is_binary);
417        assert_eq!(diffs[4].status, GitStatus::Rename);
418        assert_eq!(diffs[4].new_file_path.as_deref(), Some("apps/shared/src/scanner/file3.rs"));
419    }
420
421    #[tokio::test]
422    async fn stream_across_chunk_boundaries() {
423        let chunk1 = b"T 2305\t:100644 100644 97e7432a448baa9e97ec5e4f03c57b09b8e116ed... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/dispatcher.rs\nT 44";
424        let chunk2 = b"22\t:100644 100644 c417bf5a3fbec60b22aeab13cfa4d9439155303b... 0000000000000000000000000000000000000000... M\tapps/scan_orchestrator/src/git.rs\n";
425
426        let byte_stream = futures::stream::iter(vec![
427            Ok::<_, std::io::Error>(bytes::Bytes::from(&chunk1[..])),
428            Ok(bytes::Bytes::from(&chunk2[..])),
429        ]);
430
431        let diffs: Vec<HFFileDiff> = stream_raw_diff(byte_stream).map(|r| r.unwrap()).collect().await;
432
433        assert_eq!(diffs.len(), 2);
434        assert_eq!(diffs[0].file_path, "apps/scan_orchestrator/src/dispatcher.rs");
435        assert_eq!(diffs[1].file_path, "apps/scan_orchestrator/src/git.rs");
436    }
437}
hf_hub/repository/diff.rs

hf_hub/repository/
diff.rs
