provenant/utils/
text.rs

1// SPDX-FileCopyrightText: nexB Inc. and others
2// ScanCode is a trademark of nexB Inc.
3// SPDX-FileCopyrightText: Provenant contributors
4// SPDX-License-Identifier: Apache-2.0
5// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.
6
7use std::path::Path;
8
9const UTF8_BOM_CHAR: char = '\u{FEFF}';
10
11pub fn remove_verbatim_escape_sequences(s: &str) -> String {
12    s.replace("\\r", " ")
13        .replace("\\n", " ")
14        .replace("\\t", " ")
15}
16
17pub fn strip_utf8_bom_str(s: &str) -> &str {
18    s.strip_prefix(UTF8_BOM_CHAR).unwrap_or(s)
19}
20
21pub fn should_remove_verbatim_escape_sequences(path: &Path, is_source: bool) -> bool {
22    if is_source {
23        return true;
24    }
25
26    path.extension()
27        .and_then(|ext| ext.to_str())
28        .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "po" | "pot"))
29}
30
31#[cfg(test)]
32mod tests {
33    use super::*;
34    #[test]
35    fn test_strip_utf8_bom_str_with_bom() {
36        let s = "\u{FEFF}Hello World";
37        assert_eq!(strip_utf8_bom_str(s), "Hello World");
38    }
39
40    #[test]
41    fn test_strip_utf8_bom_str_without_bom() {
42        let s = "Hello World";
43        assert_eq!(strip_utf8_bom_str(s), "Hello World");
44    }
45
46    #[test]
47    fn test_strip_utf8_bom_str_empty() {
48        let s = "";
49        assert_eq!(strip_utf8_bom_str(s), "");
50    }
51
52    #[test]
53    fn test_strip_utf8_bom_str_only_bom() {
54        let s = "\u{FEFF}";
55        assert_eq!(strip_utf8_bom_str(s), "");
56    }
57
58    #[test]
59    fn test_bom_character_is_not_whitespace() {
60        let s = "\u{FEFF}Hello";
61        assert_ne!(s.trim(), "Hello");
62        assert_eq!(strip_utf8_bom_str(s), "Hello");
63    }
64
65    #[test]
66    fn test_remove_verbatim_escape_sequences_basic() {
67        let input = "line1\\nline2\\rline3\\tline4";
68        let output = remove_verbatim_escape_sequences(input);
69        assert_eq!(output, "line1 line2 line3 line4");
70    }
71
72    #[test]
73    fn test_remove_verbatim_escape_sequences_only_backslash_n() {
74        let input = "hello\\nworld";
75        let output = remove_verbatim_escape_sequences(input);
76        assert_eq!(output, "hello world");
77    }
78
79    #[test]
80    fn test_remove_verbatim_escape_sequences_no_escapes() {
81        let input = "normal text without escapes";
82        let output = remove_verbatim_escape_sequences(input);
83        assert_eq!(output, input);
84    }
85
86    #[test]
87    fn test_remove_verbatim_escape_sequences_actual_newline() {
88        let input = "line1\nline2";
89        let output = remove_verbatim_escape_sequences(input);
90        assert_eq!(output, "line1\nline2");
91    }
92
93    #[test]
94    fn test_remove_verbatim_escape_sequences_multiple() {
95        let input = "a\\nb\\nc\\n";
96        let output = remove_verbatim_escape_sequences(input);
97        assert_eq!(output, "a b c ");
98    }
99
100    #[test]
101    fn test_remove_verbatim_escape_sequences_options_c_sample() {
102        let input = "Try `progname --help' for more information.\\n";
103        let output = remove_verbatim_escape_sequences(input);
104        assert_eq!(output, "Try `progname --help' for more information. ");
105    }
106
107    #[test]
108    fn test_should_remove_verbatim_escape_sequences_for_source_files() {
109        assert!(should_remove_verbatim_escape_sequences(
110            Path::new("main.rs"),
111            true
112        ));
113    }
114
115    #[test]
116    fn test_should_remove_verbatim_escape_sequences_for_po_files() {
117        assert!(should_remove_verbatim_escape_sequences(
118            Path::new("locale.po"),
119            false
120        ));
121        assert!(should_remove_verbatim_escape_sequences(
122            Path::new("template.pot"),
123            false
124        ));
125    }
126
127    #[test]
128    fn test_should_not_remove_verbatim_escape_sequences_for_plain_text() {
129        assert!(!should_remove_verbatim_escape_sequences(
130            Path::new("README.txt"),
131            false
132        ));
133    }
134}
provenant/utils/text.rs

provenant/utils/
text.rs
