yaml_edit/scalar/
quoting.rs

1//! Quoting style selection, escaping and unescaping of scalar content.
2
3use super::resolve::all_digits;
4use super::{CoreScalarType, ScalarStyle, ScalarValue, INDICATORS};
5
6/// Parse a `\xNN`, `\uNNNN` or `\UNNNNNNNN` escape from `chars` (with
7/// the leading `\x`/`\u`/`\U` already consumed) and append the resulting
8/// character to `out`, falling back to the literal `\<prefix><digits>`
9/// when the digits are missing, non-hex, or outside the valid unicode
10/// range.
11fn push_unicode_escape(
12    out: &mut String,
13    chars: &mut std::iter::Peekable<std::str::Chars<'_>>,
14    prefix: char,
15    width: usize,
16) {
17    let hex_digits: String = chars.by_ref().take(width).collect();
18    // from_str_radix would accept a leading '+'
19    let decoded = if hex_digits.len() == width && hex_digits.chars().all(|c| c.is_ascii_hexdigit())
20    {
21        u32::from_str_radix(&hex_digits, 16)
22            .ok()
23            .and_then(char::from_u32)
24    } else {
25        None
26    };
27    match decoded {
28        Some(c) => out.push(c),
29        None => {
30            out.push('\\');
31            out.push(prefix);
32            out.push_str(&hex_digits);
33        }
34    }
35}
36
37/// Skip the leading white space of the lines following a line break,
38/// returning how many of those lines were empty.
39fn skip_line_prefixes(chars: &mut std::iter::Peekable<std::str::Chars<'_>>) -> usize {
40    let mut empty_lines = 0;
41    loop {
42        while chars.next_if(|c| *c == ' ' || *c == '\t').is_some() {}
43        match chars.next_if(|c| *c == '\n' || *c == '\r') {
44            Some('\r') => {
45                chars.next_if_eq(&'\n');
46            }
47            Some(_) => {}
48            None => return empty_lines,
49        }
50        empty_lines += 1;
51    }
52}
53
54impl ScalarValue {
55    /// Decode the content of a double-quoted string: process escape
56    /// sequences and fold line breaks as described in the YAML spec.
57    pub fn parse_escape_sequences(text: &str) -> String {
58        let mut result = String::with_capacity(text.len()); // Pre-allocate
59        let mut chars = text.chars().peekable();
60        // Unescaped white space is dropped if a line break follows it
61        let mut pending_white = String::new();
62
63        while let Some(ch) = chars.next() {
64            if ch == '\n' || ch == '\r' {
65                if ch == '\r' {
66                    chars.next_if_eq(&'\n');
67                }
68                pending_white.clear();
69                match skip_line_prefixes(&mut chars) {
70                    0 => result.push(' '),
71                    n => result.push_str(&"\n".repeat(n)),
72                }
73                continue;
74            }
75            if ch == ' ' || ch == '\t' {
76                pending_white.push(ch);
77                continue;
78            }
79            result.push_str(&pending_white);
80            pending_white.clear();
81            if ch == '\\' {
82                if let Some(&escaped) = chars.peek() {
83                    chars.next(); // consume the escaped character
84                    match escaped {
85                        // Standard escape sequences
86                        'n' => result.push('\n'),
87                        't' | '\t' => result.push('\t'),
88                        'r' => result.push('\r'),
89                        'b' => result.push('\x08'),
90                        'f' => result.push('\x0C'),
91                        'a' => result.push('\x07'), // bell
92                        'e' => result.push('\x1B'), // escape
93                        'v' => result.push('\x0B'), // vertical tab
94                        '0' => result.push('\0'),   // null
95                        '\\' => result.push('\\'),
96                        '"' => result.push('"'),
97                        '\'' => result.push('\''),
98                        '/' => result.push('/'),
99                        'N' => result.push('\u{85}'),
100                        '_' => result.push('\u{A0}'),
101                        'L' => result.push('\u{2028}'),
102                        'P' => result.push('\u{2029}'),
103                        ' ' => result.push(' '),
104                        '\n' | '\r' => {
105                            // An escaped line break is dropped, but empty
106                            // lines after it each still produce a line feed
107                            if escaped == '\r' {
108                                chars.next_if_eq(&'\n');
109                            }
110                            let empty_lines = skip_line_prefixes(&mut chars);
111                            result.push_str(&"\n".repeat(empty_lines));
112                        }
113                        'x' => push_unicode_escape(&mut result, &mut chars, 'x', 2),
114                        'u' => push_unicode_escape(&mut result, &mut chars, 'u', 4),
115                        'U' => push_unicode_escape(&mut result, &mut chars, 'U', 8),
116                        // Unknown escape sequence - preserve as literal
117                        _ => {
118                            result.push('\\');
119                            result.push(escaped);
120                        }
121                    }
122                } else {
123                    // Backslash at end of string
124                    result.push('\\');
125                }
126            } else {
127                result.push(ch);
128            }
129        }
130        result.push_str(&pending_white);
131
132        result
133    }
134
135    /// Detect the appropriate style for a value
136    pub(super) fn detect_style(value: &str) -> ScalarStyle {
137        // A line break has to be decided first. A quoted scalar folds its
138        // breaks into spaces, so `"x\ny\n"` written as `'x\ny\n'` reads back
139        // as `x y`, while a literal block keeps them. A literal also answers
140        // every reason needs_quoting gives, so nothing multi-line needs
141        // quotes. Only double quotes can escape a carriage return or a
142        // non-printable character, so those come first.
143        if Self::needs_double_quotes(value) {
144            return ScalarStyle::DoubleQuoted;
145        }
146        if value.contains('\n') {
147            // A block scalar holds a whitespace-only line as nothing but
148            // trailing spaces, which editors and readers commonly strip.
149            if value
150                .split('\n')
151                .any(|line| !line.is_empty() && line.trim_matches([' ', '\t']).is_empty())
152            {
153                return ScalarStyle::DoubleQuoted;
154            }
155            return ScalarStyle::Literal;
156        }
157        // Check if value needs quoting
158        if Self::needs_quoting(value) {
159            // Prefer single quotes if no single quotes in value
160            if !value.contains('\'') {
161                ScalarStyle::SingleQuoted
162            } else {
163                ScalarStyle::DoubleQuoted
164            }
165        } else {
166            ScalarStyle::Plain
167        }
168    }
169
170    /// Whether `value` can only be written as a double-quoted scalar.
171    ///
172    /// Characters outside the YAML c-printable set (spec 5.1), and the byte
173    /// order mark, may only appear escaped. A carriage return is a line
174    /// break, which plain, single-quoted and block scalars all normalise,
175    /// and NEL, LS and PS are line breaks to YAML 1.1 readers.
176    pub(super) fn needs_double_quotes(value: &str) -> bool {
177        value.chars().any(|c| {
178            !matches!(c, '\t' | '\n' | '\x20'..='\x7E' | '\u{A0}'..='\u{FFFD}' | '\u{10000}'..)
179                || matches!(c, '\u{2028}' | '\u{2029}' | '\u{FEFF}')
180        })
181    }
182
183    /// Check if a value needs quoting when treated as a string
184    pub(super) fn needs_quoting(value: &str) -> bool {
185        // Empty string needs quotes
186        if value.is_empty() {
187            return true;
188        }
189        Self::needs_double_quotes(value)
190            || Self::resolves_to_non_string(value)
191            || Self::needs_quoting_as_tagged(value)
192    }
193
194    /// Whether a plain scalar spelling `value` would be read back as
195    /// something other than a string when it carries no tag.
196    fn resolves_to_non_string(value: &str) -> bool {
197        // Anything the core schema resolves to a non-string, such as
198        // `.inf` or `.NaN`, has to be quoted to stay a string.
199        if Self::classify_plain(value) != CoreScalarType::String {
200            return true;
201        }
202
203        // Check for YAML keywords that would be misinterpreted
204        // These need quotes when we want them as strings
205        if value.eq_ignore_ascii_case("true")
206            || value.eq_ignore_ascii_case("false")
207            || value.eq_ignore_ascii_case("yes")
208            || value.eq_ignore_ascii_case("no")
209            || value.eq_ignore_ascii_case("on")
210            || value.eq_ignore_ascii_case("off")
211            || value.eq_ignore_ascii_case("null")
212            || value == "~"
213        {
214            return true;
215        }
216
217        // Also quote things that look like numbers to preserve them as strings
218        // Radix forms outside the core schema, such as `0b101` or `-0x1F`,
219        // are quoted too, as YAML 1.1 readers take them for integers.
220        let unsigned = value.strip_prefix(['+', '-']).unwrap_or(value);
221        let radix_like = ["0b", "0B", "0o", "0O", "0x", "0X"]
222            .iter()
223            .any(|p| unsigned.strip_prefix(p).is_some_and(|d| all_digits(d, 16)));
224        value.parse::<f64>().is_ok() || radix_like
225    }
226
227    /// Whether `value` needs quoting even with an explicit tag deciding its
228    /// type, because as a plain scalar it would not read back as `value`.
229    pub(super) fn needs_quoting_as_tagged(value: &str) -> bool {
230        // `-`, `?` and `:` could start a plain scalar before a safe
231        // character, but are quoted anyway, as is `<`, which is not an
232        // indicator. A plain scalar starting with `#` would read as a
233        // comment, and the value would be lost.
234        if value.starts_with(INDICATORS) || value.starts_with('<') {
235            return true;
236        }
237
238        // Document terminator (`...`) and start marker (`---`) at
239        // column 0 begin a new stream document, so they must be quoted
240        // when used as a scalar / key.
241        if value.starts_with("...") || value.starts_with("---") {
242            return true;
243        }
244
245        // Check for special characters that require quoting. `:` and `#` are
246        // only ambiguous in context, and the context differs between them:
247        // a `:` ends a key when followed by whitespace or end of input, while
248        // a `#` starts a comment when *preceded* by whitespace (YAML 1.2
249        // 6.6). So `a#b` is a fine plain scalar but `a #b` is not.
250        let mut prev: Option<char> = None;
251        let mut chars = value.chars().peekable();
252        while let Some(ch) = chars.next() {
253            match ch {
254                '&' | '*' | '!' | '|' | '\'' | '"' | '%' => return true,
255                ':' if chars.peek().map_or(true, |next| next.is_whitespace()) => {
256                    return true;
257                }
258                '#' if prev.map_or(true, char::is_whitespace) => {
259                    return true;
260                }
261                _ => {}
262            }
263            prev = Some(ch);
264        }
265
266        // Leading/trailing whitespace needs quotes
267        if value != value.trim() {
268            return true;
269        }
270
271        false
272    }
273
274    /// Convert to single-quoted string, indenting continuation lines by
275    /// `indent` spaces.
276    pub(super) fn to_single_quoted(&self, indent: usize) -> String {
277        // A single-quoted scalar folds a raw line break into a space, so a
278        // lone `\r` can only be written escaped.
279        if self.value.contains('\r') {
280            return self.to_double_quoted();
281        }
282        // Line folding trims whitespace around a break, which only double
283        // quotes can keep.
284        let white = [' ', '\t'];
285        if self
286            .value
287            .split('\n')
288            .skip(1)
289            .zip(self.value.split('\n'))
290            .any(|(next, prev)| next.starts_with(white) || prev.ends_with(white))
291        {
292            return self.to_double_quoted();
293        }
294        // A run of n breaks folds to n - 1 newlines, or a space for n = 1.
295        let mut result = String::from("'");
296        let mut prev_empty = false;
297        for (i, part) in self.value.split('\n').enumerate() {
298            if i > 0 {
299                result.push('\n');
300                if !prev_empty || i == 1 {
301                    result.push('\n');
302                }
303                if !part.is_empty() {
304                    result.push_str(&" ".repeat(indent));
305                }
306            }
307            result.push_str(&part.replace('\'', "''"));
308            prev_empty = part.is_empty();
309        }
310        result.push('\'');
311        result
312    }
313
314    /// Convert to double-quoted string
315    pub(super) fn to_double_quoted(&self) -> String {
316        let mut result = String::from("\"");
317        for ch in self.value.chars() {
318            match ch {
319                '"' => result.push_str("\\\""),
320                '\\' => result.push_str("\\\\"),
321                '\n' => result.push_str("\\n"),
322                '\r' => result.push_str("\\r"),
323                '\t' => result.push_str("\\t"),
324                '\x08' => result.push_str("\\b"),
325                '\x0C' => result.push_str("\\f"),
326                '\x07' => result.push_str("\\a"), // bell
327                '\x1B' => result.push_str("\\e"), // escape
328                '\x0B' => result.push_str("\\v"), // vertical tab
329                '\0' => result.push_str("\\0"),   // null
330                c if c.is_control() || (c as u32) > 0x7F => {
331                    // Handle Unicode characters and control characters
332                    let code_point = c as u32;
333                    if code_point <= 0xFF {
334                        result.push_str(&format!("\\x{code_point:02X}"));
335                    } else if code_point <= 0xFFFF {
336                        result.push_str(&format!("\\u{code_point:04X}"));
337                    } else {
338                        result.push_str(&format!("\\U{code_point:08X}"));
339                    }
340                }
341                c => result.push(c),
342            }
343        }
344        result.push('"');
345        result
346    }
347}
348
349#[cfg(test)]
350mod tests {
351    use super::*;
352
353    #[test]
354    fn test_escape_sequences_basic() {
355        // Test basic escape sequences
356        assert_eq!(
357            ScalarValue::parse_escape_sequences("hello\\nworld"),
358            "hello\nworld"
359        );
360        assert_eq!(
361            ScalarValue::parse_escape_sequences("tab\\there"),
362            "tab\there"
363        );
364        assert_eq!(
365            ScalarValue::parse_escape_sequences("quote\\\"test"),
366            "quote\"test"
367        );
368        assert_eq!(
369            ScalarValue::parse_escape_sequences("back\\\\slash"),
370            "back\\slash"
371        );
372        assert_eq!(
373            ScalarValue::parse_escape_sequences("return\\rtest"),
374            "return\rtest"
375        );
376    }
377
378    #[test]
379    fn test_escape_sequences_control_chars() {
380        // Test control character escapes
381        assert_eq!(ScalarValue::parse_escape_sequences("bell\\a"), "bell\x07");
382        assert_eq!(
383            ScalarValue::parse_escape_sequences("backspace\\b"),
384            "backspace\x08"
385        );
386        assert_eq!(
387            ScalarValue::parse_escape_sequences("formfeed\\f"),
388            "formfeed\x0C"
389        );
390        assert_eq!(
391            ScalarValue::parse_escape_sequences("escape\\e"),
392            "escape\x1B"
393        );
394        assert_eq!(ScalarValue::parse_escape_sequences("vtab\\v"), "vtab\x0B");
395        assert_eq!(ScalarValue::parse_escape_sequences("null\\0"), "null\0");
396        assert_eq!(ScalarValue::parse_escape_sequences("slash\\/"), "slash/");
397    }
398
399    #[test]
400    fn test_escape_sequences_unicode_x() {
401        // Test \xNN escape sequences
402        assert_eq!(ScalarValue::parse_escape_sequences("\\x41"), "A"); // 0x41 = 'A'
403        assert_eq!(ScalarValue::parse_escape_sequences("\\x7A"), "z"); // 0x7A = 'z'
404        assert_eq!(ScalarValue::parse_escape_sequences("\\x20"), " "); // 0x20 = space
405        assert_eq!(ScalarValue::parse_escape_sequences("\\xFF"), "\u{FF}"); // 0xFF = ÿ
406
407        // Test invalid hex sequences
408        assert_eq!(ScalarValue::parse_escape_sequences("\\xGH"), "\\xGH"); // Invalid hex
409        assert_eq!(ScalarValue::parse_escape_sequences("\\x4"), "\\x4"); // Incomplete
410        assert_eq!(ScalarValue::parse_escape_sequences("\\x4Z"), "\\x4Z");
411        assert_eq!(ScalarValue::parse_escape_sequences("\\xAZ"), "\\xAZ");
412        assert_eq!(ScalarValue::parse_escape_sequences("\\x+4"), "\\x+4");
413        assert_eq!(ScalarValue::parse_escape_sequences("\\u+041"), "\\u+041");
414    }
415
416    #[test]
417    fn test_escape_sequences_unicode_u() {
418        // Test \uNNNN escape sequences
419        assert_eq!(ScalarValue::parse_escape_sequences("\\u0041"), "A"); // 0x0041 = 'A'
420        assert_eq!(ScalarValue::parse_escape_sequences("\\u03B1"), "α"); // Greek alpha
421        assert_eq!(ScalarValue::parse_escape_sequences("\\u2603"), "☃"); // Snowman
422        assert_eq!(ScalarValue::parse_escape_sequences("\\u4E2D"), "中"); // Chinese character
423
424        // Test invalid sequences
425        assert_eq!(ScalarValue::parse_escape_sequences("\\uGHIJ"), "\\uGHIJ"); // Invalid hex
426        assert_eq!(ScalarValue::parse_escape_sequences("\\u041"), "\\u041"); // Incomplete
427    }
428
429    #[test]
430    fn test_escape_sequences_unicode_capital_u() {
431        // Test \UNNNNNNNN escape sequences
432        assert_eq!(ScalarValue::parse_escape_sequences("\\U00000041"), "A"); // 0x00000041 = 'A'
433        assert_eq!(ScalarValue::parse_escape_sequences("\\U0001F603"), "😃"); // Smiley emoji
434        assert_eq!(ScalarValue::parse_escape_sequences("\\U0001F4A9"), "💩"); // Pile of poo emoji
435
436        // Test invalid sequences
437        assert_eq!(
438            ScalarValue::parse_escape_sequences("\\UGHIJKLMN"),
439            "\\UGHIJKLMN"
440        ); // Invalid hex
441        assert_eq!(
442            ScalarValue::parse_escape_sequences("\\U0000004"),
443            "\\U0000004"
444        ); // Incomplete
445        assert_eq!(
446            ScalarValue::parse_escape_sequences("\\UFFFFFFFF"),
447            "\\UFFFFFFFF"
448        ); // Invalid code point
449    }
450
451    #[test]
452    fn test_escape_sequences_line_folding() {
453        // An escaped space is content; the raw line break after it folds
454        assert_eq!(
455            ScalarValue::parse_escape_sequences("line\\ \nfolding"),
456            "line  folding"
457        );
458        assert_eq!(ScalarValue::parse_escape_sequences("a\\\n  b"), "ab");
459        assert_eq!(ScalarValue::parse_escape_sequences("a\\\r\n  b"), "ab");
460        assert_eq!(
461            ScalarValue::parse_escape_sequences("escaped\\nline\\nbreak"),
462            "escaped\nline\nbreak"
463        );
464        assert_eq!(
465            ScalarValue::parse_escape_sequences("remove\\\nline\\nbreak"),
466            "removeline\nbreak"
467        );
468    }
469
470    #[test]
471    fn test_escape_sequences_mixed() {
472        // Test mixed escape sequences
473        let input = "Hello\\nWorld\\u0021\\x20\\U0001F44D";
474        let expected = "Hello\nWorld! 👍";
475        assert_eq!(ScalarValue::parse_escape_sequences(input), expected);
476
477        // Test with quotes and backslashes
478        let input = "Quote\\\"back\\\\slash\\ttab";
479        let expected = "Quote\"back\\slash\ttab";
480        assert_eq!(ScalarValue::parse_escape_sequences(input), expected);
481    }
482
483    #[test]
484    fn test_escape_sequences_yaml_specific() {
485        assert_eq!(
486            ScalarValue::parse_escape_sequences("\\N\\_\\L\\P\\\t"),
487            "\u{85}\u{A0}\u{2028}\u{2029}\t"
488        );
489    }
490
491    #[test]
492    fn test_escape_sequences_unknown() {
493        // Test unknown escape sequences are preserved
494        assert_eq!(ScalarValue::parse_escape_sequences("\\q"), "\\q");
495        assert_eq!(ScalarValue::parse_escape_sequences("\\z"), "\\z");
496        assert_eq!(ScalarValue::parse_escape_sequences("\\1"), "\\1");
497    }
498
499    #[test]
500    fn test_escape_sequences_edge_cases() {
501        // Test edge cases
502        assert_eq!(ScalarValue::parse_escape_sequences(""), "");
503        assert_eq!(ScalarValue::parse_escape_sequences("\\"), "\\");
504        assert_eq!(
505            ScalarValue::parse_escape_sequences("no escapes"),
506            "no escapes"
507        );
508        assert_eq!(ScalarValue::parse_escape_sequences("\\\\\\\\"), "\\\\");
509    }
510
511    #[test]
512    fn test_parse_escape_single_quote() {
513        // The \' escape decodes to a bare single quote.
514        assert_eq!(ScalarValue::parse_escape_sequences("a\\'b"), "a'b");
515    }
516}
yaml_edit/scalar/quoting.rs

yaml_edit/scalar/
quoting.rs
