rumdl_lib/
encoding.rs

1//! MD094 reports input that is not valid UTF-8, and the decoder that reads it.
2//!
3//! A document with a few invalid bytes (a Latin-1 `é`, a truncated sequence) is
4//! still Markdown: it is decoded lossily, each invalid sequence becomes one
5//! U+FFFD, the text is linted as usual, and MD094 reports every replacement at
6//! its position. Invalid input that looks binary (a NUL early on, or a UTF-16
7//! byte order mark) is not linted at all, because lossy text from an image or
8//! archive yields hundreds of meaningless findings from other rules.
9//!
10//! The lossy text is never written back: its U+FFFD characters stand for bytes
11//! the file really holds, so every adapter reports on it and leaves the file
12//! untouched.
13
14use crate::rule::{FixCapability, LintError, LintResult, LintWarning, Rule, RuleCategory, Severity};
15
16pub const RULE_NAME: &str = "MD094";
17
18/// How many invalid sequences are reported individually before the rest are
19/// summarized in one finding.
20pub const MAX_REPORTED: usize = 20;
21
22/// How far into the input a NUL byte marks it as binary. The same window Git
23/// uses to decide whether a file is binary.
24const BINARY_SNIFF_LEN: usize = 8000;
25
26/// One invalid UTF-8 sequence in the original bytes.
27#[derive(Debug, Clone, PartialEq, Eq)]
28pub struct InvalidSeq {
29    /// The invalid bytes as they appear in the input.
30    pub bytes: Vec<u8>,
31    /// Offset of the first invalid byte in the input.
32    pub byte_offset: usize,
33    /// Which U+FFFD of the decoded text replaced this sequence, counting every
34    /// U+FFFD, including ones the input spelled out validly. Positions are found
35    /// by this ordinal rather than by offset so they survive line-ending
36    /// normalization of the decoded text.
37    pub ordinal: usize,
38}
39
40/// The result of decoding a document's bytes.
41#[derive(Debug, PartialEq, Eq)]
42pub enum Decoded<'a> {
43    /// Valid UTF-8, borrowed unchanged.
44    Utf8(&'a str),
45    /// Invalid UTF-8 that looks like binary data or UTF-16; not linted.
46    Binary { utf16: bool },
47    /// Invalid UTF-8 decoded with one U+FFFD per invalid sequence.
48    Lossy { text: String, invalid: Vec<InvalidSeq> },
49}
50
51/// Decode a document's bytes.
52///
53/// Valid UTF-8 is never classified as binary, whatever it contains: such a file
54/// is linted exactly as it always was.
55pub fn decode(bytes: &[u8]) -> Decoded<'_> {
56    if let Ok(text) = std::str::from_utf8(bytes) {
57        return Decoded::Utf8(text);
58    }
59    let utf16 = bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]);
60    if utf16 || bytes[..bytes.len().min(BINARY_SNIFF_LEN)].contains(&0) {
61        return Decoded::Binary { utf16 };
62    }
63
64    let mut text = String::with_capacity(bytes.len() + 16);
65    let mut invalid = Vec::new();
66    let mut replacements = 0;
67    let mut offset = 0;
68    for chunk in bytes.utf8_chunks() {
69        let valid = chunk.valid();
70        text.push_str(valid);
71        replacements += valid.matches(char::REPLACEMENT_CHARACTER).count();
72        offset += valid.len();
73        let bad = chunk.invalid();
74        if !bad.is_empty() {
75            text.push(char::REPLACEMENT_CHARACTER);
76            invalid.push(InvalidSeq {
77                bytes: bad.to_vec(),
78                byte_offset: offset,
79                ordinal: replacements,
80            });
81            replacements += 1;
82            offset += bad.len();
83        }
84    }
85    Decoded::Lossy { text, invalid }
86}
87
88/// Read a Markdown file for indexing, decoding invalid UTF-8 lossily.
89///
90/// `Ok(None)` means the file is binary and has no Markdown to contribute.
91pub fn read_markdown_lossy(path: &std::path::Path) -> std::io::Result<Option<String>> {
92    let bytes = std::fs::read(path)?;
93    Ok(decode_owned(bytes))
94}
95
96/// Decode owned bytes to text, `None` for binary input.
97pub fn decode_owned(bytes: Vec<u8>) -> Option<String> {
98    match String::from_utf8(bytes) {
99        Ok(text) => Some(text),
100        Err(error) => match decode(error.as_bytes()) {
101            Decoded::Lossy { text, .. } => Some(text),
102            Decoded::Binary { .. } => None,
103            Decoded::Utf8(_) => unreachable!("from_utf8 rejected these bytes"),
104        },
105    }
106}
107
108#[derive(Debug, Clone, Default)]
109pub struct MD094InvalidEncoding;
110
111impl Rule for MD094InvalidEncoding {
112    fn name(&self) -> &'static str {
113        RULE_NAME
114    }
115    fn description(&self) -> &'static str {
116        "File is not valid UTF-8"
117    }
118    fn category(&self) -> RuleCategory {
119        RuleCategory::Other
120    }
121    fn check(&self, ctx: &crate::lint_context::LintContext) -> LintResult {
122        let Some(invalid) = ctx.invalid_utf8() else {
123            return Ok(Vec::new());
124        };
125        let mut replacements = ctx
126            .content
127            .match_indices(char::REPLACEMENT_CHARACTER)
128            .map(|(offset, _)| offset)
129            .enumerate();
130        let mut warnings = Vec::with_capacity(invalid.len());
131        for seq in invalid {
132            let Some((_, offset)) = replacements.find(|(ordinal, _)| *ordinal == seq.ordinal) else {
133                break;
134            };
135            let (line, column) = ctx.offset_to_line_col(offset);
136            warnings.push(LintWarning {
137                rule_name: Some(RULE_NAME.to_string()),
138                message: format!("Invalid UTF-8 byte sequence {} (shown as U+FFFD)", hex(&seq.bytes)),
139                line,
140                column,
141                end_line: line,
142                end_column: column + 1,
143                severity: Severity::Warning,
144                fix: None,
145            });
146        }
147        Ok(warnings)
148    }
149    fn fix_capability(&self) -> FixCapability {
150        FixCapability::Unfixable
151    }
152    fn fix(&self, ctx: &crate::lint_context::LintContext) -> Result<String, LintError> {
153        Ok(ctx.content.to_string())
154    }
155    fn as_any(&self) -> &dyn std::any::Any {
156        self
157    }
158    fn from_config(_config: &crate::config::Config) -> Box<dyn Rule> {
159        Box::new(Self)
160    }
161}
162
163fn hex(bytes: &[u8]) -> String {
164    bytes
165        .iter()
166        .map(|byte| format!("0x{byte:02X}"))
167        .collect::<Vec<_>>()
168        .join(" ")
169}
170
171/// Settle the report for a lossily decoded document.
172///
173/// Every fix is dropped: the ranges index the decoded text, whose U+FFFD
174/// characters are not the bytes on disk, so no fix can be applied or offered.
175/// Past [`MAX_REPORTED`] reportable MD094 findings, the rest collapse into one
176/// summary at the position of the first one not shown. Runs after suppression,
177/// so a suppressed sequence is neither shown nor counted.
178pub fn settle_lossy_warnings(warnings: &mut Vec<LintWarning>) {
179    for warning in warnings.iter_mut() {
180        warning.fix = None;
181    }
182    let total = warnings
183        .iter()
184        .filter(|warning| warning.rule_name.as_deref() == Some(RULE_NAME))
185        .count();
186    if total <= MAX_REPORTED {
187        return;
188    }
189    let mut seen = 0;
190    warnings.retain_mut(|warning| {
191        if warning.rule_name.as_deref() != Some(RULE_NAME) {
192            return true;
193        }
194        seen += 1;
195        if seen == MAX_REPORTED + 1 {
196            let hidden = total - MAX_REPORTED;
197            let noun = if hidden == 1 { "sequence" } else { "sequences" };
198            warning.message = format!("{hidden} more invalid UTF-8 {noun} not shown");
199        }
200        seen <= MAX_REPORTED + 1
201    });
202}
203
204/// The finding for a binary input, if the invocation reports MD094 for it.
205///
206/// `rules` is the invocation's effective rule set, so configuration and CLI
207/// rule selection (`--disable`, `--enable`) both apply; per-file-ignores and
208/// severity are read from `config`. Binary input is never parsed, so inline
209/// comments cannot suppress this finding.
210pub fn detect_binary_for_rules(
211    utf16: bool,
212    rules: &[Box<dyn Rule>],
213    config: &crate::config::Config,
214    path: Option<&std::path::Path>,
215) -> Option<LintWarning> {
216    if !rules.iter().any(|rule| rule.name() == RULE_NAME)
217        || path.is_some_and(|path| config.get_ignored_rules_for_file(path).contains(RULE_NAME))
218    {
219        return None;
220    }
221    let message = if utf16 {
222        "File appears to be UTF-16 encoded; not linted, convert it to UTF-8"
223    } else {
224        "File appears to be binary; not linted"
225    };
226    Some(LintWarning {
227        rule_name: Some(RULE_NAME.to_string()),
228        message: message.to_string(),
229        line: 1,
230        column: 1,
231        end_line: 1,
232        end_column: 1,
233        severity: config.get_rule_severity(RULE_NAME).unwrap_or(Severity::Warning),
234        fix: None,
235    })
236}
237
238/// Whether a lossily decoded document needs MD094 added back to its rule set.
239///
240/// MD094 is a guard rather than one of the outer document's rules: it says why a
241/// file rumdl cannot read is left unlinted, and that answer is owed whatever a
242/// mode is doing with the document's own rules. `--only-code-block-tools` drops
243/// the document set entirely, which would otherwise leave a lossily decoded file
244/// reporting clean while a binary file in the same invocation reports MD094 -
245/// the same question answered two ways, and the silent answer is
246/// indistinguishable from a file that is genuinely fine.
247///
248/// `selected` is the invocation's resolved selection, the gate
249/// [`detect_binary_for_rules`] applies to the other flavor. Per-file ignores,
250/// inline comments and severity stay with the lint pipeline, which is why the
251/// rule is restored to the set rather than reported from here.
252pub fn guard_missing_from_document_rules(selected: &[Box<dyn Rule>], document: &[Box<dyn Rule>]) -> bool {
253    selected.iter().any(|rule| rule.name() == RULE_NAME) && !document.iter().any(|rule| rule.name() == RULE_NAME)
254}
255
256#[cfg(test)]
257mod tests {
258    use super::*;
259    use crate::config::{Config, MarkdownFlavor};
260    use crate::lint_context::LintContext;
261
262    fn lossy(bytes: &[u8]) -> (String, Vec<InvalidSeq>) {
263        match decode(bytes) {
264            Decoded::Lossy { text, invalid } => (text, invalid),
265            other => panic!("expected lossy decoding, got {other:?}"),
266        }
267    }
268
269    fn positions(bytes: &[u8]) -> Vec<(usize, usize, String)> {
270        let (text, invalid) = lossy(bytes);
271        let text = crate::utils::normalize_line_ending(&text, crate::utils::LineEnding::Lf);
272        let ctx = LintContext::new(&text, MarkdownFlavor::Standard, None).with_invalid_utf8(&invalid);
273        MD094InvalidEncoding
274            .check(&ctx)
275            .unwrap()
276            .into_iter()
277            .map(|warning| (warning.line, warning.column, warning.message))
278            .collect()
279    }
280
281    #[test]
282    fn valid_utf8_is_borrowed_even_when_it_contains_nul() {
283        assert_eq!(decode(b"# Title\n"), Decoded::Utf8("# Title\n"));
284        assert_eq!(decode(&[0; 64]), Decoded::Utf8("\0".repeat(64).leak()));
285    }
286
287    #[test]
288    fn invalid_sequences_keep_their_bytes_and_offsets() {
289        let (text, invalid) = lossy(b"caf\xE9 \xE2\x82 ok\xFF");
290        assert_eq!(text, "caf\u{FFFD} \u{FFFD} ok\u{FFFD}");
291        let found: Vec<_> = invalid.iter().map(|seq| (seq.bytes.clone(), seq.byte_offset)).collect();
292        assert_eq!(found, vec![(vec![0xE9], 3), (vec![0xE2, 0x82], 5), (vec![0xFF], 10)]);
293    }
294
295    #[test]
296    fn truncated_sequence_at_end_of_input_is_one_finding() {
297        let (text, invalid) = lossy(b"end \xF0\x9F\x98");
298        assert_eq!(text, "end \u{FFFD}");
299        assert_eq!(invalid.len(), 1);
300        assert_eq!(invalid[0].bytes, vec![0xF0, 0x9F, 0x98]);
301    }
302
303    #[test]
304    fn overlong_and_surrogate_encodings_are_invalid() {
305        // Each maximal invalid prefix is replaced separately, as Rust's own
306        // lossy decoding does.
307        let (_, overlong) = lossy(b"a\xC0\xAFb");
308        assert_eq!(overlong.len(), 2);
309        let (_, surrogate) = lossy(b"a\xED\xA0\x80b");
310        assert_eq!(surrogate.len(), 3);
311        assert_eq!(lossy(b"a\xED\xA0\x80b").0, String::from_utf8_lossy(b"a\xED\xA0\x80b"));
312    }
313
314    #[test]
315    fn literal_replacement_characters_are_not_findings() {
316        let mut input = "\u{FFFD} valid\nbad \u{FFFD} ".as_bytes().to_vec();
317        input.push(0xE9);
318        let found = positions(&input);
319        assert_eq!(found.len(), 1);
320        assert_eq!((found[0].0, found[0].1), (2, 7));
321    }
322
323    #[test]
324    fn positions_are_line_and_character_column() {
325        let found = positions(&["é line one\nsecond ".as_bytes(), b"\xE9 here"].concat());
326        assert_eq!(
327            found,
328            vec![(2, 8, "Invalid UTF-8 byte sequence 0xE9 (shown as U+FFFD)".into())]
329        );
330        let found = positions(b"x\xE2\x82y");
331        assert_eq!(found[0].2, "Invalid UTF-8 byte sequence 0xE2 0x82 (shown as U+FFFD)");
332    }
333
334    #[test]
335    fn positions_survive_crlf_and_long_input() {
336        let mut input = Vec::new();
337        for line in 1..=400 {
338            input.extend_from_slice(format!("Line {line} with some padding text\r\n").as_bytes());
339        }
340        assert!(input.len() > 8192);
341        input.extend_from_slice(b"tail \xE9\r\n");
342        assert_eq!(
343            positions(&input),
344            vec![(401, 6, "Invalid UTF-8 byte sequence 0xE9 (shown as U+FFFD)".into())]
345        );
346    }
347
348    #[test]
349    fn nul_within_the_sniff_window_marks_binary() {
350        let mut at_edge = vec![b'a'; BINARY_SNIFF_LEN + 10];
351        at_edge[0] = 0xE9;
352        at_edge[BINARY_SNIFF_LEN - 1] = 0;
353        assert_eq!(decode(&at_edge), Decoded::Binary { utf16: false });
354        at_edge[BINARY_SNIFF_LEN - 1] = b'a';
355        at_edge[BINARY_SNIFF_LEN] = 0;
356        assert!(matches!(decode(&at_edge), Decoded::Lossy { .. }));
357    }
358
359    #[test]
360    fn utf16_byte_order_marks_are_reported_as_utf16() {
361        assert_eq!(decode(b"\xFF\xFE#\0 \0"), Decoded::Binary { utf16: true });
362        assert_eq!(decode(b"\xFE\xFF\0#\0 "), Decoded::Binary { utf16: true });
363        assert_eq!(decode(b"#\0 \0T\0\xE9\0"), Decoded::Binary { utf16: false });
364    }
365
366    #[test]
367    fn decode_owned_matches_decode() {
368        assert_eq!(decode_owned(b"ok".to_vec()).as_deref(), Some("ok"));
369        assert_eq!(decode_owned(b"caf\xE9".to_vec()).as_deref(), Some("caf\u{FFFD}"));
370        assert_eq!(decode_owned(b"\xE9\0".to_vec()), None);
371    }
372
373    #[test]
374    fn no_findings_without_decoder_data() {
375        let ctx = LintContext::new("bad \u{FFFD}", MarkdownFlavor::Standard, None);
376        assert!(MD094InvalidEncoding.check(&ctx).unwrap().is_empty());
377    }
378
379    fn finding(line: usize, rule: &str) -> LintWarning {
380        LintWarning {
381            rule_name: Some(rule.to_string()),
382            message: format!("{rule} at {line}"),
383            line,
384            column: 1,
385            end_line: line,
386            end_column: 2,
387            severity: Severity::Warning,
388            fix: Some(crate::rule::Fix::new(0..1, String::new())),
389        }
390    }
391
392    #[test]
393    fn settling_caps_md094_and_drops_every_fix() {
394        let mut warnings: Vec<_> = (1..=25).map(|line| finding(line, RULE_NAME)).collect();
395        warnings.insert(3, finding(3, "MD009"));
396        settle_lossy_warnings(&mut warnings);
397        assert!(warnings.iter().all(|warning| warning.fix.is_none()));
398        let md094: Vec<_> = warnings
399            .iter()
400            .filter(|warning| warning.rule_name.as_deref() == Some(RULE_NAME))
401            .collect();
402        assert_eq!(md094.len(), MAX_REPORTED + 1);
403        assert_eq!(md094[MAX_REPORTED].line, 21);
404        assert_eq!(md094[MAX_REPORTED].message, "5 more invalid UTF-8 sequences not shown");
405        assert_eq!(warnings.len(), MAX_REPORTED + 2);
406    }
407
408    #[test]
409    fn settling_leaves_up_to_the_cap_alone() {
410        let mut warnings: Vec<_> = (1..=MAX_REPORTED).map(|line| finding(line, RULE_NAME)).collect();
411        settle_lossy_warnings(&mut warnings);
412        assert_eq!(warnings.len(), MAX_REPORTED);
413        assert!(warnings.iter().all(|warning| !warning.message.contains("more")));
414    }
415
416    #[test]
417    fn binary_finding_follows_rule_selection_and_severity() {
418        let config = Config::default();
419        let md094: Vec<Box<dyn Rule>> = vec![Box::new(MD094InvalidEncoding)];
420        let found = detect_binary_for_rules(false, &md094, &config, None).unwrap();
421        assert_eq!(found.message, "File appears to be binary; not linted");
422        assert_eq!(found.severity, Severity::Warning);
423        assert!(
424            detect_binary_for_rules(true, &md094, &config, None)
425                .unwrap()
426                .message
427                .contains("UTF-16")
428        );
429
430        // A rule set without MD094, as `--disable MD094` or `--enable MD009`
431        // produces, reports nothing.
432        assert!(detect_binary_for_rules(false, &[], &config, None).is_none());
433    }
434}
rumdl_lib/encoding.rs

rumdl_lib/
encoding.rs
