provenant/utils/
text.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// SPDX-FileCopyrightText: Provenant contributors
3// SPDX-License-Identifier: Apache-2.0
4// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
5
6use std::path::Path;
7
8const UTF8_BOM_CHAR: char = '\u{FEFF}';
9
10pub fn remove_verbatim_escape_sequences(s: &str) -> String {
11    s.replace("\\r", " ")
12        .replace("\\n", " ")
13        .replace("\\t", " ")
14}
15
16pub fn strip_utf8_bom_str(s: &str) -> &str {
17    s.strip_prefix(UTF8_BOM_CHAR).unwrap_or(s)
18}
19
20pub fn should_remove_verbatim_escape_sequences(path: &Path, is_source: bool) -> bool {
21    if is_source {
22        return true;
23    }
24
25    path.extension()
26        .and_then(|ext| ext.to_str())
27        .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "po" | "pot"))
28}
29
30#[cfg(test)]
31mod tests {
32    use super::*;
33    #[test]
34    fn test_strip_utf8_bom_str_with_bom() {
35        let s = "\u{FEFF}Hello World";
36        assert_eq!(strip_utf8_bom_str(s), "Hello World");
37    }
38
39    #[test]
40    fn test_strip_utf8_bom_str_without_bom() {
41        let s = "Hello World";
42        assert_eq!(strip_utf8_bom_str(s), "Hello World");
43    }
44
45    #[test]
46    fn test_strip_utf8_bom_str_empty() {
47        let s = "";
48        assert_eq!(strip_utf8_bom_str(s), "");
49    }
50
51    #[test]
52    fn test_strip_utf8_bom_str_only_bom() {
53        let s = "\u{FEFF}";
54        assert_eq!(strip_utf8_bom_str(s), "");
55    }
56
57    #[test]
58    fn test_bom_character_is_not_whitespace() {
59        let s = "\u{FEFF}Hello";
60        assert_ne!(s.trim(), "Hello");
61        assert_eq!(strip_utf8_bom_str(s), "Hello");
62    }
63
64    #[test]
65    fn test_remove_verbatim_escape_sequences_basic() {
66        let input = "line1\\nline2\\rline3\\tline4";
67        let output = remove_verbatim_escape_sequences(input);
68        assert_eq!(output, "line1 line2 line3 line4");
69    }
70
71    #[test]
72    fn test_remove_verbatim_escape_sequences_only_backslash_n() {
73        let input = "hello\\nworld";
74        let output = remove_verbatim_escape_sequences(input);
75        assert_eq!(output, "hello world");
76    }
77
78    #[test]
79    fn test_remove_verbatim_escape_sequences_no_escapes() {
80        let input = "normal text without escapes";
81        let output = remove_verbatim_escape_sequences(input);
82        assert_eq!(output, input);
83    }
84
85    #[test]
86    fn test_remove_verbatim_escape_sequences_actual_newline() {
87        let input = "line1\nline2";
88        let output = remove_verbatim_escape_sequences(input);
89        assert_eq!(output, "line1\nline2");
90    }
91
92    #[test]
93    fn test_remove_verbatim_escape_sequences_multiple() {
94        let input = "a\\nb\\nc\\n";
95        let output = remove_verbatim_escape_sequences(input);
96        assert_eq!(output, "a b c ");
97    }
98
99    #[test]
100    fn test_remove_verbatim_escape_sequences_options_c_sample() {
101        let input = "Try `progname --help' for more information.\\n";
102        let output = remove_verbatim_escape_sequences(input);
103        assert_eq!(output, "Try `progname --help' for more information. ");
104    }
105
106    #[test]
107    fn test_should_remove_verbatim_escape_sequences_for_source_files() {
108        assert!(should_remove_verbatim_escape_sequences(
109            Path::new("main.rs"),
110            true
111        ));
112    }
113
114    #[test]
115    fn test_should_remove_verbatim_escape_sequences_for_po_files() {
116        assert!(should_remove_verbatim_escape_sequences(
117            Path::new("locale.po"),
118            false
119        ));
120        assert!(should_remove_verbatim_escape_sequences(
121            Path::new("template.pot"),
122            false
123        ));
124    }
125
126    #[test]
127    fn test_should_not_remove_verbatim_escape_sequences_for_plain_text() {
128        assert!(!should_remove_verbatim_escape_sequences(
129            Path::new("README.txt"),
130            false
131        ));
132    }
133}
provenant/utils/text.rs

provenant/utils/
text.rs
