oneyaml-syntax 0.1.2

Lossless YAML 1.2 decoder, parser, event stream, and syntax tree underneath oneyaml.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
//! Lexical layer: character classes (YAML 1.2.2 §5) and single-line scanners
//! for indicators, properties (§6.9), and plain scalar lines (§7.3.3).
//!
//! Every function takes the source and a byte offset on a `char` boundary and
//! returns offsets on `char` boundaries. None of them allocate or recurse, and
//! each scans forward only, so the parser stays O(n).

use crate::event::{TagForm, TriviaKind};

/// The byte at `i`, if any.
pub(super) fn byte(src: &str, i: usize) -> Option<u8> {
    src.as_bytes().get(i).copied()
}

/// The `char` starting at byte `i`, if any.
pub(super) fn char_at(src: &str, i: usize) -> Option<char> {
    src.get(i..).and_then(|s| s.chars().next())
}

/// `s-white` (production 33).
pub(super) const fn is_white(b: u8) -> bool {
    b == b' ' || b == b'\t'
}

/// `b-char` (production 26).
pub(super) const fn is_break(b: u8) -> bool {
    b == b'\n' || b == b'\r'
}

/// White space, a line break, or the end of input at `i`.
pub(super) fn blank_or_end(src: &str, i: usize) -> bool {
    byte(src, i).is_none_or(|b| is_white(b) || is_break(b))
}

/// A line break or the end of input at `i`.
pub(super) fn break_or_end(src: &str, i: usize) -> bool {
    byte(src, i).is_none_or(is_break)
}

/// Length of the line break at `i` (`\r\n` is one break); `0` if none.
pub(super) fn break_len(src: &str, i: usize) -> usize {
    match byte(src, i) {
        Some(b'\r') if byte(src, i.saturating_add(1)) == Some(b'\n') => 2,
        Some(b'\n' | b'\r') => 1,
        _ => 0,
    }
}

/// The offset of the first line break at or after `i`, or `src.len()`.
pub(super) fn line_end(src: &str, i: usize) -> usize {
    src.get(i..)
        .and_then(|rest| rest.find(['\n', '\r']))
        .map_or(src.len(), |k| i.saturating_add(k))
}

/// The offset after the run of spaces and tabs starting at `i`.
pub(super) fn skip_white(src: &str, i: usize) -> usize {
    let run = src
        .as_bytes()
        .get(i..)
        .map_or(0, |rest| rest.iter().take_while(|b| is_white(**b)).count());
    i.saturating_add(run)
}

/// The first byte order mark in `from..to`, if any.
pub(super) fn first_mark(src: &str, from: usize, to: usize) -> Option<usize> {
    let text = src.get(from..to)?;
    text.find('\u{feff}').map(|k| from.saturating_add(k))
}

/// A byte order mark starts at `i`.
pub(super) fn mark_at(src: &str, i: usize) -> bool {
    src.get(i..).is_some_and(|r| r.starts_with('\u{feff}'))
}

/// `c-printable` (production 1).
pub(super) const fn is_printable(c: char) -> bool {
    matches!(c,
        '\t' | '\n' | '\r' | '\x20'..='\x7e' | '\u{85}' | '\u{a0}'..='\u{d7ff}'
        | '\u{e000}'..='\u{fffd}' | '\u{10000}'..)
}

/// `nb-json` (production 2): allowed inside quoted scalars.
pub(super) const fn is_nb_json(c: char) -> bool {
    c == '\t' || c >= '\x20'
}

/// `ns-char` (production 34): printable, not white space, not a break, not
/// a byte order mark.
pub(super) const fn is_ns_char(c: char) -> bool {
    is_printable(c) && !matches!(c, ' ' | '\t' | '\n' | '\r' | '\u{feff}')
}

/// `c-flow-indicator` (production 23).
pub(super) const fn is_flow_indicator(c: char) -> bool {
    matches!(c, ',' | '[' | ']' | '{' | '}')
}

/// `c-indicator` (production 22).
const fn is_indicator(c: char) -> bool {
    matches!(
        c,
        '-' | '?'
            | ':'
            | ','
            | '['
            | ']'
            | '{'
            | '}'
            | '#'
            | '&'
            | '*'
            | '!'
            | '|'
            | '>'
            | '\''
            | '"'
            | '%'
            | '@'
            | '`'
    )
}

/// `ns-plain-safe(c)` (productions 127-129) for the char at `i`; block
/// context (`flow == false`) allows flow indicators.
fn plain_safe_at(src: &str, i: usize, flow: bool) -> bool {
    char_at(src, i).is_some_and(|c| is_ns_char(c) && !(flow && is_flow_indicator(c)))
}

/// `ns-plain-first(c)` (production 126) at `i`.
pub(super) fn is_plain_first(src: &str, i: usize, flow: bool) -> bool {
    match char_at(src, i) {
        Some('-' | '?' | ':') => plain_safe_at(src, i.saturating_add(1), flow),
        Some(c) => is_ns_char(c) && !is_indicator(c),
        None => false,
    }
}

/// An indicator `c` at `i` followed by white space, a break, or the end of
/// input (e.g. `- `, `? `, `: ` in block context).
pub(super) fn indicator_at(src: &str, i: usize, c: u8) -> bool {
    byte(src, i) == Some(c) && blank_or_end(src, i.saturating_add(1))
}

/// A document marker (`---` or `...`, productions 203 and 204) at `i`, which
/// must be the start of a line: the three characters followed by white space,
/// a break, or the end of input.
pub(super) fn marker_at(src: &str, i: usize, marker: &str) -> bool {
    src.get(i..)
        .is_some_and(|rest| rest.starts_with(marker) && blank_or_end(src, i.saturating_add(3)))
}

/// Either document marker at `i`.
pub(super) fn any_marker_at(src: &str, i: usize) -> bool {
    marker_at(src, i, "---") || marker_at(src, i, "...")
}

/// End of the plain scalar text on the line starting at `start`
/// (`nb-ns-plain-in-line`, production 132, after `ns-plain-first`), with
/// trailing white space excluded. Stops before `: `, ` #`, a line break, a
/// non-printable character, and (in flow context) flow indicators.
pub(super) fn plain_line_end(src: &str, start: usize, flow: bool) -> usize {
    line_text_end(src, start, flow, false)
}

/// [`plain_line_end`] in block context, except that `: ` does not end the
/// text: the rest of the line up to a comment (` #`), a line break, or a
/// non-printable character. For `lenient_inline_colons` and for the
/// quoting fix of `INLINE_COLON_IN_PLAIN_VALUE`.
pub(super) fn plain_line_end_lenient(src: &str, start: usize) -> usize {
    line_text_end(src, start, false, true)
}

fn line_text_end(src: &str, start: usize, flow: bool, colons: bool) -> usize {
    let mut i = start;
    let mut end = start;
    while let Some(c) = char_at(src, i) {
        match c {
            ' ' | '\t' => {
                i = i.saturating_add(1);
                continue;
            }
            // ns-plain-char (production 130): `:` only before a safe char.
            ':' if !colons && !plain_safe_at(src, i.saturating_add(1), flow) => break,
            // `#` only directly after a non-space character.
            '#' if i > start
                && src
                    .as_bytes()
                    .get(i.saturating_sub(1))
                    .is_some_and(|b| is_white(*b)) =>
            {
                break;
            }
            c if !is_ns_char(c) || (flow && is_flow_indicator(c)) => break,
            _ => {}
        }
        i = i.saturating_add(c.len_utf8());
        end = i;
    }
    end
}

/// End of an anchor or alias name starting at `i` (`ns-anchor-name`,
/// production 103: `ns-char` minus flow indicators).
pub(super) fn anchor_end(src: &str, i: usize) -> usize {
    let mut j = i;
    while let Some(c) = char_at(src, j) {
        if !is_ns_char(c) || is_flow_indicator(c) {
            break;
        }
        j = j.saturating_add(c.len_utf8());
    }
    j
}

/// `ns-word-char` (production 38).
const fn is_word_char(b: u8) -> bool {
    b.is_ascii_alphanumeric() || b == b'-'
}

/// Length of one `ns-uri-char` (production 39) at `i` (3 for `%XX`), or 0.
/// `tag_char` excludes `!` and flow indicators (`ns-tag-char`, production 40).
fn uri_char_len(src: &str, i: usize, tag_char: bool) -> usize {
    let Some(b) = byte(src, i) else { return 0 };
    let hex = |k: usize| byte(src, i.saturating_add(k)).is_some_and(|h| h.is_ascii_hexdigit());
    match b {
        b'%' if hex(1) && hex(2) => 3,
        b'!' | b',' | b'[' | b']' if tag_char => 0,
        _ if is_word_char(b) => 1,
        b'#' | b';' | b'/' | b'?' | b':' | b'@' | b'&' | b'=' | b'+' | b'$' | b'_' | b'.'
        | b'~' | b'*' | b'\'' | b'(' | b')' | b'!' | b',' | b'[' | b']' => 1,
        _ => 0,
    }
}

/// The offset after a run of URI (or tag) characters starting at `i`.
pub(super) fn uri_end(src: &str, i: usize, tag_char: bool) -> usize {
    let mut j = i;
    loop {
        let n = uri_char_len(src, j, tag_char);
        if n == 0 {
            return j;
        }
        j = j.saturating_add(n);
    }
}

/// A scanned tag property (`c-ns-tag-property`, production 97): byte ranges
/// of the handle and suffix as written.
#[derive(Clone, Copy, Debug)]
pub(super) struct TagScan {
    pub(super) form: TagForm,
    pub(super) handle: (usize, usize),
    pub(super) suffix: (usize, usize),
    pub(super) end: usize,
}

/// Scans a tag starting with `!` at `start`. `None` when malformed.
pub(super) fn scan_tag(src: &str, start: usize) -> Option<TagScan> {
    let after = start.saturating_add(1);
    match byte(src, after) {
        // c-verbatim-tag (production 98).
        Some(b'<') => {
            let from = after.saturating_add(1);
            let to = uri_end(src, from, false);
            (to > from && byte(src, to) == Some(b'>')).then_some(TagScan {
                form: TagForm::Verbatim,
                handle: (start, start),
                suffix: (from, to),
                end: to.saturating_add(1),
            })
        }
        // c-ns-shorthand-tag with the secondary handle `!!` (production 99).
        Some(b'!') => {
            let from = after.saturating_add(1);
            let to = uri_end(src, from, true);
            (to > from).then_some(TagScan {
                form: TagForm::Shorthand,
                handle: (start, from),
                suffix: (from, to),
                end: to,
            })
        }
        _ => {
            let words = src
                .as_bytes()
                .get(after..)
                .map_or(0, |r| r.iter().take_while(|b| is_word_char(**b)).count());
            let word_end = after.saturating_add(words);
            // Named handle `!word!` (production 92).
            let (handle_end, from) = if words > 0 && byte(src, word_end) == Some(b'!') {
                let h = word_end.saturating_add(1);
                (h, h)
            } else {
                (after, after)
            };
            let to = uri_end(src, from, true);
            if to == from {
                // c-non-specific-tag (production 100): a lone `!`.
                return (handle_end == after).then_some(TagScan {
                    form: TagForm::NonSpecific,
                    handle: (start, after),
                    suffix: (after, after),
                    end: after,
                });
            }
            Some(TagScan {
                form: TagForm::Shorthand,
                handle: (start, handle_end),
                suffix: (from, to),
                end: to,
            })
        }
    }
}

/// The tag URI `prefix` + `suffix` with its `%XX` escapes decoded, built in
/// one allocation. `None` when the decoded bytes are not UTF-8.
///
/// §6.8.2.2: the prefix is part of the URI, so its escapes
/// are decoded too, and UTF-8 is checked on the whole result — a multi-byte
/// character may be split across prefix and suffix (`%C3` + `%A9` is `é`).
/// Neither part ends inside an escape (`ns-uri-char`, production 39, needs
/// both hex digits), so this equals decoding the concatenated text.
pub(super) fn percent_decode_concat(prefix: &str, suffix: &str) -> Option<String> {
    let mut out = Vec::with_capacity(prefix.len().saturating_add(suffix.len()));
    out.extend(decoded(prefix));
    out.extend(decoded(suffix));
    String::from_utf8(out).ok()
}

/// The bytes of `s` with its `%XX` escapes decoded, one at a time (no
/// allocation).
pub(super) fn decoded(s: &str) -> impl Iterator<Item = u8> + '_ {
    let bytes = s.as_bytes();
    let mut i = 0usize;
    core::iter::from_fn(move || {
        let b = *bytes.get(i)?;
        let hex = bytes
            .get(i.saturating_add(1)..i.saturating_add(3))
            .and_then(|h| core::str::from_utf8(h).ok())
            .and_then(|h| u8::from_str_radix(h, 16).ok());
        match (b, hex) {
            (b'%', Some(v)) => {
                i = i.saturating_add(3);
                Some(v)
            }
            _ => {
                i = i.saturating_add(1);
                Some(b)
            }
        }
    })
}

/// Incremental UTF-8 validation (RFC 3629) of decoded tag bytes: the
/// state between two bytes, so a `%TAG` prefix is validated once and each
/// tag continues from where its prefix ended.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct Utf8 {
    /// Continuation bytes still owed by the unfinished sequence.
    need: u8,
    /// The range the next continuation byte must lie in.
    lo: u8,
    hi: u8,
}

impl Utf8 {
    /// Between characters.
    pub(super) const START: Self = Self {
        need: 0,
        lo: 0x80,
        hi: 0xBF,
    };

    const fn owing(need: u8, lo: u8, hi: u8) -> Self {
        Self { need, lo, hi }
    }

    /// The state after byte `b`; `None` when no UTF-8 continues with it.
    pub(super) fn push(self, b: u8) -> Option<Self> {
        if self.need > 0 {
            return (self.lo..=self.hi)
                .contains(&b)
                .then(|| Self::owing(self.need.saturating_sub(1), 0x80, 0xBF));
        }
        Some(match b {
            0x00..=0x7F => Self::START,
            0xC2..=0xDF => Self::owing(1, 0x80, 0xBF),
            0xE0 => Self::owing(2, 0xA0, 0xBF),
            0xED => Self::owing(2, 0x80, 0x9F),
            0xE1..=0xEF => Self::owing(2, 0x80, 0xBF),
            0xF0 => Self::owing(3, 0x90, 0xBF),
            0xF1..=0xF3 => Self::owing(3, 0x80, 0xBF),
            0xF4 => Self::owing(3, 0x80, 0x8F),
            _ => return None,
        })
    }

    /// No sequence is left unfinished.
    pub(super) const fn complete(self) -> bool {
        self.need == 0
    }
}

/// A URI part read on: its decoded length and the UTF-8 state after it
/// (`None`: it already holds a byte no UTF-8 continues with).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct UriRead {
    pub(super) len: usize,
    pub(super) utf8: Option<Utf8>,
}

impl UriRead {
    /// Nothing read yet.
    pub(super) const EMPTY: Self = Self {
        len: 0,
        utf8: Some(Utf8::START),
    };

    /// This state continued over the decoded bytes of `s`: O(`s`).
    pub(super) fn then(self, s: &str) -> Self {
        let mut out = self;
        for b in decoded(s) {
            out.len = out.len.saturating_add(1);
            out.utf8 = out.utf8.and_then(|u| u.push(b));
        }
        out
    }

    /// The whole URI read is UTF-8.
    pub(super) fn is_utf8(self) -> bool {
        self.utf8.is_some_and(Utf8::complete)
    }
}

/// A `%TAG` prefix, read once at its directive: where it
/// starts, and its decoded length and UTF-8 state, so that a tag using it
/// is checked in O(its suffix).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct TagPrefix {
    pub(super) start: usize,
    pub(super) read: UriRead,
}

impl TagPrefix {
    /// The prefix starting at `start` (`None`: empty, which a declared
    /// prefix never is). O(prefix), once per directive.
    pub(super) fn read(src: &str, start: usize) -> Option<Self> {
        let text = tag_prefix_at(src, start)?;
        Some(Self {
            start,
            read: UriRead::EMPTY.then(text),
        })
    }
}

/// The `%TAG` prefix (`ns-tag-prefix`, production 93) starting at `p`, as
/// written; `None` when empty (a declared prefix never is).
pub(super) fn tag_prefix_at(src: &str, p: usize) -> Option<&str> {
    src.get(p..uri_end(src, p, false)).filter(|t| !t.is_empty())
}

/// `#` at `i` starts a comment: preceded by white space, a break, a byte
/// order mark, or the start of input (production 75).
pub(super) fn comment_at(src: &str, i: usize) -> bool {
    byte(src, i) == Some(b'#')
        && src.get(..i).is_some_and(|before| {
            matches!(
                before.chars().next_back(),
                None | Some(' ' | '\t' | '\n' | '\r' | '\u{feff}')
            )
        })
}

/// Lexes the first trivia event of `src[i..end]`, a run of white space,
/// line breaks, comments, and byte order marks that the trivia scanners
/// (`s-separate`, `s-l-comments`, productions 79-80) accepted: its kind and
/// end. The parser queues pending trivia as such a run and lexes it only as
/// the consumer pulls events, so the events are exactly the ones the
/// scanners saw: white space is one maximal run, a comment runs to the end
/// of its line, a break is one break.
///
/// `None` if `src[i]` cannot start trivia (or `i..end` is empty): only
/// trivia is ever queued as a run, so that is a broken internal invariant,
/// which the caller reports as `Y9000`.
pub(super) fn lex_trivia(src: &str, i: usize, end: usize) -> Option<(TriviaKind, usize)> {
    if i >= end {
        return None;
    }
    let (kind, stop) = match byte(src, i) {
        Some(b' ' | b'\t') => (TriviaKind::Whitespace, skip_white(src, i)),
        Some(b'\n' | b'\r') => (TriviaKind::LineBreak, i.saturating_add(break_len(src, i))),
        Some(b'#') => (TriviaKind::Comment, line_end(src, i)),
        _ if src.get(i..).is_some_and(|r| r.starts_with('\u{feff}')) => (
            TriviaKind::ByteOrderMark,
            i.saturating_add('\u{feff}'.len_utf8()),
        ),
        _ => return None,
    };
    Some((kind, stop.min(end).max(i.saturating_add(1)).min(end)))
}

/// The implicit-key limit (productions 154-155): at most 1024 characters,
/// counting properties, content, and white space before the `:`.
pub(super) const IMPLICIT_KEY_MAX_CHARS: usize = 1024;

/// `src[start..end]` is longer than [`IMPLICIT_KEY_MAX_CHARS`] characters.
/// O(1024): short byte ranges need no counting.
pub(super) fn key_too_long(src: &str, start: usize, end: usize) -> bool {
    src.get(start..end).is_some_and(|key| {
        key.len() > IMPLICIT_KEY_MAX_CHARS && key.chars().nth(IMPLICIT_KEY_MAX_CHARS).is_some()
    })
}

/// `!`, `!!`, or `!word!` (`c-tag-handle`, production 89).
pub(super) fn valid_handle(h: &str) -> bool {
    if h == "!" || h == "!!" {
        return true;
    }
    h.strip_prefix('!')
        .and_then(|r| r.strip_suffix('!'))
        .is_some_and(|w| !w.is_empty() && w.bytes().all(is_word_char))
}

#[cfg(test)]
mod tests {
    #![allow(clippy::unwrap_used, clippy::indexing_slicing)]

    use super::*;

    #[test]
    fn breaks_and_lines() {
        assert_eq!(break_len("a\r\nb", 1), 2);
        assert_eq!(break_len("a\rb", 1), 1);
        assert_eq!(break_len("a\nb", 0), 0);
        assert_eq!(line_end("ab\ncd", 0), 2);
        assert_eq!(line_end("ab", 1), 2);
        assert_eq!(skip_white("a \t b", 1), 4);
    }

    #[test]
    fn plain_lines() {
        let s = "a b: c";
        assert_eq!(plain_line_end(s, 0, false), 3);
        assert_eq!(plain_line_end("a:b #c", 0, false), 3);
        assert_eq!(plain_line_end("a#b  ", 0, false), 3);
        assert_eq!(plain_line_end("a,b]", 0, false), 4);
        assert_eq!(plain_line_end("a,b]", 0, true), 1);
        assert!(is_plain_first("-a", 0, false));
        assert!(!is_plain_first("- a", 0, false));
        assert!(!is_plain_first("[a", 0, false));
        assert!(!is_plain_first("@a", 0, false));
        assert_eq!(plain_line_end_lenient("a: b: c  # d", 0), 7);
        assert_eq!(plain_line_end_lenient("a:\tb:", 0), 5);
        assert_eq!(plain_line_end_lenient("a: [b, c] :\nx", 0), 11);
        assert_eq!(plain_line_end_lenient("a: b\u{1}c", 0), 4);
    }

    #[test]
    fn markers_and_indicators() {
        assert!(marker_at("--- a", 0, "---"));
        assert!(marker_at("...", 0, "..."));
        assert!(!marker_at("---a", 0, "---"));
        assert!(indicator_at("- a", 0, b'-'));
        assert!(indicator_at("-", 0, b'-'));
        assert!(!indicator_at("-a", 0, b'-'));
    }

    #[test]
    fn tags() {
        let t = scan_tag("!!str x", 0).unwrap();
        assert_eq!(
            (t.form, t.handle, t.suffix, t.end),
            (TagForm::Shorthand, (0, 2), (2, 5), 5)
        );
        let t = scan_tag("!e!foo", 0).unwrap();
        assert_eq!((t.handle, t.suffix), ((0, 3), (3, 6)));
        let t = scan_tag("!foo", 0).unwrap();
        assert_eq!((t.handle, t.suffix), ((0, 1), (1, 4)));
        let t = scan_tag("! a", 0).unwrap();
        assert_eq!((t.form, t.end), (TagForm::NonSpecific, 1));
        let t = scan_tag("!<tag:a,b> x", 0).unwrap();
        assert_eq!((t.form, t.suffix, t.end), (TagForm::Verbatim, (2, 9), 10));
        assert!(scan_tag("!<> x", 0).is_none());
        assert!(scan_tag("!! x", 0).is_none());
        assert!(scan_tag("!e! x", 0).is_none());
        assert_eq!(percent_decode_concat("p:", "a%21b").unwrap(), "p:a!b");
        assert_eq!(percent_decode_concat("", "%4").unwrap(), "%4");
        assert!(percent_decode_concat("", "%ff").is_none());
        // The prefix is decoded too, and UTF-8 is checked on the
        // whole URI.
        assert_eq!(percent_decode_concat("p:%21", "a").unwrap(), "p:!a");
        assert_eq!(percent_decode_concat("p:%C3", "%A9").unwrap(), "p:é");
        assert!(percent_decode_concat("%FF", "x").is_none());
        assert!(percent_decode_concat("p:%C3", "x").is_none());
        assert_eq!(tag_prefix_at("%TAG !e! a:%21 ", 9), Some("a:%21"));
        assert_eq!(tag_prefix_at("x ", 1), None);
        // A prefix read once, continued by each suffix.
        let p = TagPrefix::read("%TAG !e! a:%C3 ", 9).unwrap();
        assert_eq!(p.read.len, 3);
        assert!(!p.read.is_utf8() && p.read.utf8.is_some());
        assert!(p.read.then("%A9x").is_utf8());
        assert_eq!(p.read.then("%A9x").len, 5);
        assert!(!p.read.then("x").is_utf8());
        assert!(TagPrefix::read("x", 1).is_none());
        assert!(valid_handle("!e!") && !valid_handle("!e") && !valid_handle("!!!"));
    }

    /// The incremental validator agrees with `std::str::from_utf8` on
    /// every 1- and 2-byte sequence, and on 3- and 4-byte ones over the
    /// boundary continuation bytes.
    #[test]
    fn utf8_validation_matches_std() {
        let valid = |bytes: &[u8]| {
            let mut u = Some(Utf8::START);
            for b in bytes {
                u = u.and_then(|u| u.push(*b));
            }
            u.is_some_and(Utf8::complete)
        };
        let edges = [0x00, 0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0, 0xFF];
        for a in 0..=255u8 {
            assert_eq!(valid(&[a]), core::str::from_utf8(&[a]).is_ok(), "{a:x}");
            for b in 0..=255u8 {
                let two = [a, b];
                assert_eq!(valid(&two), core::str::from_utf8(&two).is_ok());
                for c in edges {
                    let three = [a, b, c];
                    assert_eq!(valid(&three), core::str::from_utf8(&three).is_ok());
                    for d in edges {
                        let four = [a, b, c, d];
                        assert_eq!(valid(&four), core::str::from_utf8(&four).is_ok());
                    }
                }
            }
        }
    }

    #[test]
    fn anchors() {
        assert_eq!(anchor_end("&a: b", 1), 3);
        assert_eq!(anchor_end("&a,b", 1), 2);
        assert_eq!(anchor_end("&a\u{a0}b c", 1), 5);
        assert_eq!(anchor_end("&a\u{feff}", 1), 2);
    }
}