parsec_encoding/
prescan.rs1use encoding_rs::{Encoding, UTF_8, UTF_16BE, UTF_16LE, WINDOWS_1252, X_USER_DEFINED};
9
10pub fn prescan(bytes: &[u8]) -> Option<&'static Encoding> {
20 scan(bytes).or_else(|| xml_encoding(bytes))
23}
24
25pub(crate) fn scan(b: &[u8]) -> Option<&'static Encoding> {
29 if b.starts_with(&[0x3C, 0, 0x3F, 0, 0x78, 0]) {
30 return Some(UTF_16LE);
31 }
32 if b.starts_with(&[0, 0x3C, 0, 0x3F, 0, 0x78]) {
33 return Some(UTF_16BE);
34 }
35 let mut p = 0;
36 loop {
37 let rest = b.get(p..).filter(|r| !r.is_empty())?;
38 if rest.starts_with(b"<!--") {
39 p += 2 + find(&rest[2..], b"-->")? + 2;
42 } else if rest.len() >= 6
43 && rest[0] == b'<'
44 && rest[1..5].eq_ignore_ascii_case(b"meta")
45 && (is_space(rest[5]) || rest[5] == b'/')
46 {
47 p += 5;
48 if let Some(found) = meta(b, &mut p)? {
49 return Some(found);
50 }
51 } else if rest[0] == b'<' && starts_tag(&rest[1..]) {
52 p += rest
53 .iter()
54 .position(|&c| matches!(c, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'>'))?;
55 while attribute(b, &mut p)?.is_some() {}
56 } else if rest.starts_with(b"<!") || rest.starts_with(b"</") || rest.starts_with(b"<?") {
57 p += 1 + rest[1..].iter().position(|&c| c == b'>')?;
58 }
59 p += 1;
60 }
61}
62
63fn starts_tag(after: &[u8]) -> bool {
65 let after = after.strip_prefix(b"/").unwrap_or(after);
66 after.first().is_some_and(u8::is_ascii_alphabetic)
67}
68
69fn meta(b: &[u8], p: &mut usize) -> Option<Option<&'static Encoding>> {
72 let mut seen: Vec<Vec<u8>> = Vec::new();
73 let mut got_pragma = false;
74 let mut need_pragma: Option<bool> = None;
75 let mut charset: Option<Option<&'static Encoding>> = None;
78 while let Some((name, value)) = attribute(b, p)? {
79 if seen.contains(&name) {
80 continue;
81 }
82 match name.as_slice() {
83 b"http-equiv" if value == b"content-type" => got_pragma = true,
84 b"content" if charset.is_none() => {
85 if let Some(found) = charset_in_content(&value) {
86 charset = Some(Some(found));
87 need_pragma = Some(true);
88 }
89 }
90 b"charset" => {
91 charset = Some(Encoding::for_label(&value));
92 need_pragma = Some(false);
93 }
94 _ => {}
95 }
96 seen.push(name);
97 }
98 Some(match (need_pragma, charset) {
99 (Some(need), Some(Some(found))) if !need || got_pragma => Some(as_declared(found)),
100 _ => None,
101 })
102}
103
104pub(crate) fn as_declared(e: &'static Encoding) -> &'static Encoding {
107 if e == UTF_16BE || e == UTF_16LE {
108 UTF_8
109 } else if e == X_USER_DEFINED {
110 WINDOWS_1252
111 } else {
112 e
113 }
114}
115
116fn is_space(c: u8) -> bool {
117 matches!(c, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ')
118}
119
120#[allow(clippy::type_complexity)]
123fn attribute(b: &[u8], p: &mut usize) -> Option<Option<(Vec<u8>, Vec<u8>)>> {
124 while is_space(*b.get(*p)?) || b[*p] == b'/' {
125 *p += 1;
126 }
127 if b[*p] == b'>' {
128 return Some(None);
129 }
130 let mut name = Vec::new();
131 let mut value = Vec::new();
132 loop {
133 match *b.get(*p)? {
134 b'=' if !name.is_empty() => {
135 *p += 1;
136 break;
137 }
138 c if is_space(c) => {
139 while is_space(*b.get(*p)?) {
140 *p += 1;
141 }
142 if b[*p] != b'=' {
143 return Some(Some((name, value)));
144 }
145 *p += 1;
146 break;
147 }
148 b'/' | b'>' => return Some(Some((name, value))),
149 c => name.push(c.to_ascii_lowercase()),
150 }
151 *p += 1;
152 }
153 while is_space(*b.get(*p)?) {
154 *p += 1;
155 }
156 match b[*p] {
157 quote @ (b'"' | b'\'') => loop {
158 *p += 1;
159 let c = *b.get(*p)?;
160 if c == quote {
161 *p += 1;
162 return Some(Some((name, value)));
163 }
164 value.push(c.to_ascii_lowercase());
165 },
166 b'>' => return Some(Some((name, value))),
167 c => {
168 value.push(c.to_ascii_lowercase());
169 *p += 1;
170 }
171 }
172 loop {
173 match *b.get(*p)? {
174 c if is_space(c) || c == b'>' => return Some(Some((name, value))),
175 c => value.push(c.to_ascii_lowercase()),
176 }
177 *p += 1;
178 }
179}
180
181fn xml_encoding(b: &[u8]) -> Option<&'static Encoding> {
183 if !b.starts_with(b"<?xml") {
184 return None;
185 }
186 let end = b.iter().position(|&c| c == b'>')?;
187 let mut p = find(&b[..end], b"encoding")? + b"encoding".len();
188 while *b.get(p)? <= 0x20 {
189 p += 1;
190 }
191 if b[p] != b'=' {
192 return None;
193 }
194 p += 1;
195 while *b.get(p)? <= 0x20 {
196 p += 1;
197 }
198 let quote = b[p];
199 if quote != b'"' && quote != b'\'' {
200 return None;
201 }
202 p += 1;
203 let len = b[p..].iter().position(|&c| c == quote)?;
204 let name = &b[p..p + len];
205 if name.iter().any(|&c| c <= 0x20) {
206 return None;
207 }
208 let found = Encoding::for_label(name)?;
209 Some(if found == UTF_16BE || found == UTF_16LE {
210 UTF_8
211 } else {
212 found
213 })
214}
215
216pub(crate) fn charset_in_content(s: &[u8]) -> Option<&'static Encoding> {
219 let mut p = 0;
220 loop {
221 p += s[p..]
222 .windows(7)
223 .position(|w| w.eq_ignore_ascii_case(b"charset"))?
224 + 7;
225 while s.get(p).is_some_and(|c| c.is_ascii_whitespace()) {
226 p += 1;
227 }
228 if s.get(p) != Some(&b'=') {
229 continue;
230 }
231 p += 1;
232 while s.get(p).is_some_and(|c| c.is_ascii_whitespace()) {
233 p += 1;
234 }
235 let rest = &s[p..];
236 return match *rest.first()? {
237 quote @ (b'"' | b'\'') => {
238 let len = rest[1..].iter().position(|&c| c == quote)?;
239 Encoding::for_label(&rest[1..1 + len])
240 }
241 _ => {
242 let len = rest
243 .iter()
244 .position(|&c| c.is_ascii_whitespace() || c == b';')
245 .unwrap_or(rest.len());
246 Encoding::for_label(&rest[..len])
247 }
248 };
249 }
250}
251
252fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
253 haystack.windows(needle.len()).position(|w| w == needle)
254}
255
256#[cfg(test)]
257mod tests {
258 use super::*;
259 use encoding_rs::{EUC_JP, ISO_8859_2, SHIFT_JIS, WINDOWS_1251};
260
261 #[test]
262 fn meta_charset_and_pragma() {
263 assert_eq!(prescan(b"<meta charset='euc-jp'>"), Some(EUC_JP));
264 assert_eq!(
265 prescan(b"<meta http-equiv=Content-Type content='text/html; charset=windows-1251'>"),
266 Some(WINDOWS_1251)
267 );
268 assert_eq!(prescan(b"<meta content='charset=windows-1251'>"), None);
270 assert_eq!(
272 prescan(b"<meta content='charset=iso-8859-2' http-equiv=content-type>"),
273 Some(ISO_8859_2)
274 );
275 }
276
277 #[test]
278 fn charset_failure_blocks_content() {
279 let b = b"<meta charset=nonsense http-equiv=content-type content='charset=utf-8'>";
280 assert_eq!(prescan(b), None);
281 }
282
283 #[test]
284 fn duplicates_and_comments() {
285 assert_eq!(
286 prescan(b"<meta charset=sjis charset=euc-jp>"),
287 Some(SHIFT_JIS)
288 );
289 assert_eq!(prescan(b"<!-- <meta charset=euc-jp> --><p>"), None);
290 assert_eq!(prescan(b"<!--><meta charset=euc-jp>"), Some(EUC_JP));
292 assert_eq!(prescan(b"<a title='<meta charset=euc-jp>'>x"), None);
294 }
295
296 #[test]
297 fn utf16_and_user_defined_are_mapped() {
298 assert_eq!(prescan(b"<meta charset=utf-16le>"), Some(UTF_8));
299 assert_eq!(
300 prescan(b"<meta charset=x-user-defined>"),
301 Some(WINDOWS_1252)
302 );
303 assert_eq!(prescan(b"<\0?\0x\0m\0l\0"), Some(UTF_16LE));
304 assert_eq!(prescan(b"\0<\0?\0x\0m\0l"), Some(UTF_16BE));
305 }
306
307 #[test]
308 fn running_out_falls_back_to_xml() {
309 assert_eq!(prescan(b"<meta charset=euc-jp"), None);
310 assert_eq!(
311 prescan(b"<?xml version='1.0' encoding='Shift_JIS'?><p"),
312 Some(SHIFT_JIS)
313 );
314 assert_eq!(
315 prescan(b"<?xml version='1.0' encoding='utf-16'?><p"),
316 Some(UTF_8)
317 );
318 assert_eq!(prescan(b"<?xml encoding = \"euc-jp\"?>"), Some(EUC_JP));
319 assert_eq!(prescan(b"<?xml encoding='euc-jp'?><p>"), Some(EUC_JP));
321 }
322
323 #[test]
324 fn content_extraction() {
325 assert_eq!(
326 charset_in_content(b"text/html; charset=sjis"),
327 Some(SHIFT_JIS)
328 );
329 assert_eq!(
330 charset_in_content(b"charsetcharset = \"euc-jp\""),
331 Some(EUC_JP)
332 );
333 assert_eq!(charset_in_content(b"charset='euc-jp"), None);
334 assert_eq!(charset_in_content(b"charset="), None);
335 assert_eq!(charset_in_content(b"CHARSET=sjis;x"), Some(SHIFT_JIS));
336 }
337
338 #[test]
339 fn every_prefix_runs_out_cleanly() {
340 let page: &[u8] = b"<?xml encoding = 'x'?><!-- c --><a b='c' d=e f><meta \
341 http-equiv=content-type content=\"charset = 'sjis'\" charset>";
342 for end in 0..=page.len() {
343 prescan(&page[..end]);
344 }
345 assert_eq!(prescan(page), None);
348 let without = b"<meta http-equiv=content-type content=\"charset = 'sjis'\">";
349 assert_eq!(prescan(without), Some(SHIFT_JIS));
350 }
351}