jj_lib/
eol.rs

1// Copyright 2025 The Jujutsu Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use bstr::ByteSlice as _;
16use futures::AsyncRead;
17use futures::AsyncReadExt as _;
18use futures::io::Cursor;
19
20use crate::config::ConfigGetError;
21use crate::settings::UserSettings;
22
23fn is_binary(bytes: &[u8]) -> bool {
24    // TODO(06393993): align the algorithm with git so that the git config autocrlf
25    // users won't see different decisions on whether a file is binary and needs to
26    // perform EOL conversion.
27    let mut bytes = bytes.iter().peekable();
28    while let Some(byte) = bytes.next() {
29        match *byte {
30            b'\0' => return true,
31            b'\r' if bytes.peek() != Some(&&b'\n') => {
32                return true;
33            }
34            _ => {}
35        }
36    }
37    false
38}
39
40#[derive(Clone)]
41pub(crate) struct TargetEolStrategy {
42    eol_conversion_mode: EolConversionMode,
43}
44
45impl TargetEolStrategy {
46    pub(crate) fn new(eol_conversion_mode: EolConversionMode) -> Self {
47        Self {
48            eol_conversion_mode,
49        }
50    }
51
52    /// The limit to probe for whether the file is binary is 8KB.
53    /// All files strictly smaller than the limit are always
54    /// evaluated correctly and in full.
55    /// Files larger than the limit - or with ambiguous content at the limit -
56    /// are potentially misclassified.
57    const PROBE_LIMIT: u64 = 8 << 10;
58
59    /// Peek into the first [`TargetEolStrategy::PROBE_LIMIT`] bytes of content
60    /// to determine if it is binary data.
61    ///
62    /// Peeked data is stored in `peek`.
63    async fn probe_for_binary(
64        mut contents: impl AsyncRead + Unpin,
65        peek: &mut Vec<u8>,
66    ) -> Result<bool, std::io::Error> {
67        (&mut contents)
68            .take(Self::PROBE_LIMIT)
69            .read_to_end(peek)
70            .await?;
71
72        // The probe limit may have sliced a CRLF sequence, which would cause
73        // misclassification as binary.
74        let slice_to_check = if peek.get(Self::PROBE_LIMIT as usize - 1) == Some(&b'\r') {
75            &peek[0..Self::PROBE_LIMIT as usize - 1]
76        } else {
77            peek
78        };
79
80        Ok(is_binary(slice_to_check))
81    }
82
83    pub(crate) async fn convert_eol_for_snapshot<'a>(
84        &self,
85        mut contents: impl AsyncRead + Send + Unpin + 'a,
86    ) -> Result<Box<dyn AsyncRead + Send + Unpin + 'a>, std::io::Error> {
87        match self.eol_conversion_mode {
88            EolConversionMode::None => Ok(Box::new(contents)),
89            EolConversionMode::Input | EolConversionMode::InputOutput => {
90                let mut peek = vec![];
91                let target_eol = if Self::probe_for_binary(&mut contents, &mut peek).await? {
92                    TargetEol::PassThrough
93                } else {
94                    TargetEol::Lf
95                };
96                let peek = Cursor::new(peek);
97                let contents = peek.chain(contents);
98                convert_eol(contents, target_eol).await
99            }
100        }
101    }
102
103    pub(crate) async fn convert_eol_for_update<'a>(
104        &self,
105        mut contents: impl AsyncRead + Send + Unpin + 'a,
106    ) -> Result<Box<dyn AsyncRead + Send + Unpin + 'a>, std::io::Error> {
107        match self.eol_conversion_mode {
108            EolConversionMode::None | EolConversionMode::Input => Ok(Box::new(contents)),
109            EolConversionMode::InputOutput => {
110                let mut peek = vec![];
111                let target_eol = if Self::probe_for_binary(&mut contents, &mut peek).await? {
112                    TargetEol::PassThrough
113                } else {
114                    TargetEol::Crlf
115                };
116                let peek = Cursor::new(peek);
117                let contents = peek.chain(contents);
118                convert_eol(contents, target_eol).await
119            }
120        }
121    }
122}
123
124/// Configuring auto-converting CRLF line endings into LF when you add a file to
125/// the backend, and vice versa when it checks out code onto your filesystem.
126#[derive(Debug, PartialEq, Eq, Copy, Clone, serde::Deserialize)]
127#[serde(rename_all(deserialize = "kebab-case"))]
128pub enum EolConversionMode {
129    /// Do not perform EOL conversion.
130    None,
131    /// Only perform the CRLF to LF EOL conversion when writing to the backend
132    /// store from the file system.
133    Input,
134    /// Perform CRLF to LF EOL conversion when writing to the backend store from
135    /// the file system and LF to CRLF EOL conversion when writing to the file
136    /// system from the backend store.
137    InputOutput,
138}
139
140impl EolConversionMode {
141    /// Try to create the [`EolConversionMode`] based on the
142    /// `working-copy.eol-conversion` setting in the [`UserSettings`].
143    pub fn try_from_settings(user_settings: &UserSettings) -> Result<Self, ConfigGetError> {
144        user_settings.get("working-copy.eol-conversion")
145    }
146}
147
148#[derive(Clone, Copy, Debug, PartialEq, Eq)]
149enum TargetEol {
150    Lf,
151    Crlf,
152    PassThrough,
153}
154
155async fn convert_eol<'a>(
156    mut input: impl AsyncRead + Send + Unpin + 'a,
157    target_eol: TargetEol,
158) -> Result<Box<dyn AsyncRead + Send + Unpin + 'a>, std::io::Error> {
159    let eol = match target_eol {
160        TargetEol::PassThrough => {
161            return Ok(Box::new(input));
162        }
163        TargetEol::Lf => b"\n".as_slice(),
164        TargetEol::Crlf => b"\r\n".as_slice(),
165    };
166
167    let mut contents = vec![];
168    input.read_to_end(&mut contents).await?;
169    let lines = contents.lines_with_terminator();
170    let mut res = Vec::<u8>::with_capacity(contents.len());
171    fn trim_last_eol(input: &[u8]) -> Option<&[u8]> {
172        input
173            .strip_suffix(b"\r\n")
174            .or_else(|| input.strip_suffix(b"\n"))
175    }
176    for line in lines {
177        if let Some(line) = trim_last_eol(line) {
178            res.extend_from_slice(line);
179            // If the line ends with an EOL, we should append the target EOL.
180            res.extend_from_slice(eol);
181        } else {
182            // If the line doesn't end with an EOL, we don't append the EOL. This can happen
183            // on the last line.
184            res.extend_from_slice(line);
185        }
186    }
187    Ok(Box::new(Cursor::new(res)))
188}
189
190#[cfg(test)]
191mod tests {
192    use std::error::Error;
193    use std::pin::Pin;
194    use std::task::Poll;
195
196    use test_case::test_case;
197
198    use super::*;
199
200    #[tokio::main(flavor = "current_thread")]
201    #[test_case(b"a\n", TargetEol::PassThrough, b"a\n"; "LF text with no EOL conversion")]
202    #[test_case(b"a\r\n", TargetEol::PassThrough, b"a\r\n"; "CRLF text with no EOL conversion")]
203    #[test_case(b"a", TargetEol::PassThrough, b"a"; "no EOL text with no EOL conversion")]
204    #[test_case(b"a\n", TargetEol::Crlf, b"a\r\n"; "LF text with CRLF EOL conversion")]
205    #[test_case(b"a\r\n", TargetEol::Crlf, b"a\r\n"; "CRLF text with CRLF EOL conversion")]
206    #[test_case(b"a", TargetEol::Crlf, b"a"; "no EOL text with CRLF conversion")]
207    #[test_case(b"", TargetEol::Crlf, b""; "empty text with CRLF EOL conversion")]
208    #[test_case(b"a\nb", TargetEol::Crlf, b"a\r\nb"; "text ends without EOL with CRLF EOL conversion")]
209    #[test_case(b"a\n", TargetEol::Lf, b"a\n"; "LF text with LF EOL conversion")]
210    #[test_case(b"a\r\n", TargetEol::Lf, b"a\n"; "CRLF text with LF EOL conversion")]
211    #[test_case(b"a", TargetEol::Lf, b"a"; "no EOL text with LF conversion")]
212    #[test_case(b"", TargetEol::Lf, b""; "empty text with LF EOL conversion")]
213    #[test_case(b"a\r\nb", TargetEol::Lf, b"a\nb"; "text ends without EOL with LF EOL conversion")]
214    async fn test_eol_conversion(input: &[u8], target_eol: TargetEol, expected_output: &[u8]) {
215        let mut input = input;
216        let mut output = vec![];
217        convert_eol(&mut input, target_eol)
218            .await
219            .expect("Failed to call convert_eol")
220            .read_to_end(&mut output)
221            .await
222            .expect("Failed to read from the result");
223        assert_eq!(output, expected_output);
224    }
225
226    struct ErrorReader(Option<std::io::Error>);
227
228    impl ErrorReader {
229        fn new(error: std::io::Error) -> Self {
230            Self(Some(error))
231        }
232    }
233
234    impl AsyncRead for ErrorReader {
235        fn poll_read(
236            mut self: Pin<&mut Self>,
237            _cx: &mut std::task::Context<'_>,
238            _buf: &mut [u8],
239        ) -> Poll<std::io::Result<usize>> {
240            if let Some(e) = self.0.take() {
241                return Poll::Ready(Err(e));
242            }
243            Poll::Ready(Ok(0))
244        }
245    }
246
247    #[tokio::main(flavor = "current_thread")]
248    #[test_case(TargetEol::PassThrough; "no EOL conversion")]
249    #[test_case(TargetEol::Lf; "LF EOL conversion")]
250    #[test_case(TargetEol::Crlf; "CRLF EOL conversion")]
251    async fn test_eol_convert_eol_read_error(target_eol: TargetEol) {
252        let message = "test error";
253        let error_reader = ErrorReader::new(std::io::Error::other(message));
254        let mut output = vec![];
255        // TODO: use TryFutureExt::and_then and async closure after we upgrade to 1.85.0
256        // or later.
257        let err = match convert_eol(error_reader, target_eol).await {
258            Ok(mut reader) => reader.read_to_end(&mut output).await,
259            Err(e) => Err(e),
260        }
261        .expect_err("should fail");
262        let has_expected_error_message = (0..)
263            .scan(Some(&err as &(dyn Error + 'static)), |err, _| {
264                let current_err = err.take()?;
265                *err = current_err.source();
266                Some(current_err)
267            })
268            .any(|e| e.to_string() == message);
269        assert!(
270            has_expected_error_message,
271            "should have expected error message: {message}"
272        );
273    }
274
275    fn test_probe_limit_input_crlf() -> [u8; TargetEolStrategy::PROBE_LIMIT as usize + 1] {
276        let mut arr = [b'a'; TargetEolStrategy::PROBE_LIMIT as usize + 1];
277        let crlf = b"\r\n";
278        arr[100..102].copy_from_slice(crlf);
279        arr[500..502].copy_from_slice(crlf);
280        arr[1000..1002].copy_from_slice(crlf);
281        arr[4090..4092].copy_from_slice(crlf);
282        arr[TargetEolStrategy::PROBE_LIMIT as usize - 1
283            ..TargetEolStrategy::PROBE_LIMIT as usize + 1]
284            .copy_from_slice(crlf);
285        arr
286    }
287
288    fn test_probe_limit_input_lf() -> Vec<u8> {
289        test_probe_limit_input_crlf().replace(b"\r\n", b"\n")
290    }
291
292    #[tokio::main(flavor = "current_thread")]
293    #[test_case(TargetEolStrategy {
294          eol_conversion_mode: EolConversionMode::None,
295      }, b"\r\n", b"\r\n"; "none settings")]
296    #[test_case(TargetEolStrategy {
297          eol_conversion_mode: EolConversionMode::Input,
298      }, b"\r\n", b"\n"; "input settings text input")]
299    #[test_case(TargetEolStrategy {
300          eol_conversion_mode: EolConversionMode::InputOutput,
301      }, b"\r\n", b"\n"; "input output settings text input")]
302    #[test_case(TargetEolStrategy {
303          eol_conversion_mode: EolConversionMode::Input,
304      }, b"\0\r\n", b"\0\r\n"; "input settings binary input")]
305    #[test_case(TargetEolStrategy {
306          eol_conversion_mode: EolConversionMode::InputOutput,
307      }, b"\0\r\n", b"\0\r\n"; "input output settings binary input with NUL")]
308    #[test_case(TargetEolStrategy {
309          eol_conversion_mode: EolConversionMode::InputOutput,
310      }, b"\r\r\n", b"\r\r\n"; "input output settings binary input with lone CR")]
311    #[test_case(TargetEolStrategy {
312          eol_conversion_mode: EolConversionMode::Input,
313      }, &[0; 20 << 10], &[0; 20 << 10]; "input settings long binary input")]
314    #[test_case(TargetEolStrategy {
315          eol_conversion_mode: EolConversionMode::Input,
316      }, &test_probe_limit_input_crlf(), &test_probe_limit_input_lf(); "input settings with CRLF on probe boundary")]
317    async fn test_eol_strategy_convert_eol_for_snapshot(
318        strategy: TargetEolStrategy,
319        contents: &[u8],
320        expected_output: &[u8],
321    ) {
322        let mut actual_output = vec![];
323        strategy
324            .convert_eol_for_snapshot(contents)
325            .await
326            .unwrap()
327            .read_to_end(&mut actual_output)
328            .await
329            .unwrap();
330        assert_eq!(actual_output, expected_output);
331    }
332
333    #[tokio::main(flavor = "current_thread")]
334    #[test_case(TargetEolStrategy {
335          eol_conversion_mode: EolConversionMode::None,
336      }, b"\n", b"\n"; "none settings")]
337    #[test_case(TargetEolStrategy {
338          eol_conversion_mode: EolConversionMode::Input,
339      }, b"\n", b"\n"; "input settings")]
340    #[test_case(TargetEolStrategy {
341          eol_conversion_mode: EolConversionMode::InputOutput,
342      }, b"\n", b"\r\n"; "input output settings text input")]
343    #[test_case(TargetEolStrategy {
344          eol_conversion_mode: EolConversionMode::InputOutput,
345      }, b"\0\n", b"\0\n"; "input output settings binary input")]
346    #[test_case(TargetEolStrategy {
347          eol_conversion_mode: EolConversionMode::Input,
348      }, &[0; 20 << 10], &[0; 20 << 10]; "input output settings long binary input")]
349    async fn test_eol_strategy_convert_eol_for_update(
350        strategy: TargetEolStrategy,
351        contents: &[u8],
352        expected_output: &[u8],
353    ) {
354        let mut actual_output = vec![];
355        strategy
356            .convert_eol_for_update(contents)
357            .await
358            .unwrap()
359            .read_to_end(&mut actual_output)
360            .await
361            .unwrap();
362        assert_eq!(actual_output, expected_output);
363    }
364}
jj_lib/eol.rs

jj_lib/
eol.rs
