1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
//! Encoding normalization: BOM stripping and UTF-16 transcoding.
//!
//! Apply `normalize_encoding` on raw file bytes BEFORE calling `is_binary()`.
//! UTF-16 LE/BE files have embedded null bytes that would otherwise trigger
//! the binary-skip heuristic; transcoding to UTF-8 removes those nulls and
//! makes the content indexable.
use std::borrow::Cow;
/// Normalize raw file bytes for indexing and search.
///
/// - Strips the UTF-8 BOM (`EF BB BF`) and returns the remainder.
/// - Detects UTF-16 LE BOM (`FF FE`) and transcodes to UTF-8.
/// - Detects UTF-16 BE BOM (`FE FF`) and transcodes to UTF-8.
/// - Returns all other content as `Cow::Borrowed` (zero copy).
///
/// **Must be called before `is_binary()`**: UTF-16 files have null bytes at
/// every other character position and would be silently skipped without this.
///
/// Emits a debug log if a UTF-16 file has an odd byte count after the BOM
/// (truncated on disk). The incomplete trailing byte is decoded as U+FFFD,
/// matching the WHATWG UTF-16 decoder (and thus ripgrep); dropping it caused
/// false-positive divergences under `-x`.
pub(crate) fn normalize_encoding(content: &[u8]) -> Cow<'_, [u8]> {
if let Some(rest) = content.strip_prefix(b"\xEF\xBB\xBF") {
return Cow::Borrowed(rest);
}
if let Some(rest) = content.strip_prefix(b"\xFF\xFE") {
return Cow::Owned(decode_utf16(rest, u16::from_le_bytes));
}
if let Some(rest) = content.strip_prefix(b"\xFE\xFF") {
return Cow::Owned(decode_utf16(rest, u16::from_be_bytes));
}
Cow::Borrowed(content)
}
fn decode_utf16(bytes: &[u8], from_bytes: fn([u8; 2]) -> u16) -> Vec<u8> {
let chunks = bytes.chunks_exact(2);
let truncated = !chunks.remainder().is_empty();
if truncated {
log::debug!(
"UTF-16 file has odd byte count ({} bytes after BOM); trailing byte decoded as U+FFFD",
bytes.len()
);
}
let mut out = char::decode_utf16(chunks.map(|c| from_bytes([c[0], c[1]])))
.map(|r| r.unwrap_or('\u{FFFD}'))
.collect::<String>();
if truncated {
// WHATWG UTF-16 decoders (encoding_rs, hence ripgrep) emit a
// replacement character for an incomplete final code unit rather than
// dropping it. Completes the same lossy policy already applied to lone
// surrogates above (`unwrap_or('\u{FFFD}')`) and restores rg parity:
// dropping the byte let `-x parse` match st's "parse" but not rg's
// "parse\u{FFFD}".
out.push('\u{FFFD}');
}
out.into_bytes()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn no_bom_returns_borrowed() {
let content = b"fn main() {}";
let result = normalize_encoding(content);
assert!(
matches!(result, Cow::Borrowed(_)),
"plain UTF-8 must return Cow::Borrowed (zero copy)"
);
assert_eq!(result.as_ref(), content);
}
#[test]
fn utf8_bom_stripped() {
let input = b"\xEF\xBB\xBFfn main() {}";
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), b"fn main() {}");
}
#[test]
fn utf8_bom_only_file() {
let result = normalize_encoding(b"\xEF\xBB\xBF");
assert_eq!(result.as_ref(), b"");
}
#[test]
fn utf16_le_ascii_transcoded() {
// "hi\n" in UTF-16 LE with BOM: FF FE 68 00 69 00 0A 00
let input: &[u8] = b"\xFF\xFEh\x00i\x00\n\x00";
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), b"hi\n");
}
#[test]
fn utf16_be_ascii_transcoded() {
// "hi\n" in UTF-16 BE with BOM: FE FF 00 68 00 69 00 0A
let input: &[u8] = b"\xFE\xFF\x00h\x00i\x00\n";
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), b"hi\n");
}
#[test]
fn utf16_le_non_bmp_replacement_char() {
// Lone high surrogate (D800) -> U+FFFD (EF BF BD in UTF-8)
let input: &[u8] = b"\xFF\xFE\x00\xD8"; // BOM + lone surrogate
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), "\u{FFFD}".as_bytes());
}
#[test]
fn utf16_le_odd_byte_trailing_truncated() {
// Incomplete final code unit -> U+FFFD, matching rg (encoding_rs).
let input: &[u8] = b"\xFF\xFEh\x00i"; // BOM + "h" + lone byte
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), "h\u{FFFD}".as_bytes());
}
#[test]
fn utf16_be_odd_byte_trailing_truncated() {
// Mirrors oracle fixture repro_e1c1603c26349124: BE BOM + "parse" +
// lone 0x0D. rg decodes the dangling byte as U+FFFD (not \r), so `-x
// parse` must NOT match. Regression guard for the st<->rg divergence.
let input: &[u8] = b"\xFE\xFF\x00p\x00a\x00r\x00s\x00e\x0D";
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), "parse\u{FFFD}".as_bytes());
}
#[test]
fn empty_content_returns_borrowed() {
let result = normalize_encoding(b"");
assert!(matches!(result, Cow::Borrowed(_)));
assert_eq!(result.as_ref(), b"");
}
#[test]
fn utf16_le_source_code() {
let src = "fn main() {}";
let utf16le: Vec<u8> = src.encode_utf16().flat_map(|u| u.to_le_bytes()).collect();
let mut input = vec![0xFF, 0xFE]; // LE BOM
input.extend_from_slice(&utf16le);
let result = normalize_encoding(&input);
assert_eq!(result.as_ref(), src.as_bytes());
}
#[test]
fn utf16_le_odd_byte_verbose_warning() {
// verbose=true should not panic; warning goes to stderr (not testable here,
// but we verify the output is still correct).
let input: &[u8] = b"\xFF\xFEh\x00i"; // BOM + "h" + lone byte
let result = normalize_encoding(input);
assert_eq!(result.as_ref(), "h\u{FFFD}".as_bytes());
}
}