powerliners 0.1.3

// vim:fileencoding=utf-8:noet
//! Port of `powerline/lint/markedjson/scanner.py`.
//!
//! YAML-style scanner state machine. The Python source produces
//! tokens of type STREAM-START / STREAM-END / FLOW-SEQUENCE-START /
//! FLOW-MAPPING-START / FLOW-SEQUENCE-END / FLOW-MAPPING-END /
//! FLOW-ENTRY / KEY / VALUE / SCALAR consumed by the parser
//! (`parser.rs`) and composer (`composer.rs`).
//!
//! Rust port surfaces:
//!   - `ScannerError` (MarkedError subclass)
//!   - `SimpleKey` struct
//!   - `hexdigits_set()` accessor for the Python `set(hexdigits)`
//!     module-level constant
//!   - `Scanner` struct skeleton with the core state machine fields
//!     (done / flow_level / tokens_taken / possible_simple_keys /
//!     allow_simple_key) and the public check_token / peek_token /
//!     get_token / fetch_stream_start interfaces
//!   - `dispatch_fetch_for(char)` — the dispatch table at py:140-187
//!     mapping the next character to the appropriate fetch_* token-
//!     type discriminator
//!
//! The heavy scan_* methods (scan_flow_scalar / scan_plain /
//! scan_to_next_token / scan_flow_scalar_non_spaces / scan_flow_
//! scalar_spaces) are deferred since they need the Reader-backed
//! source buffer + the Reader peek/forward interface.

// from __future__ import (unicode_literals, division, absolute_import, print_function)  // py:2
// from string import hexdigits                     // py:4
// from powerline.lint.markedjson.error import MarkedError                                 // py:6
// from powerline.lint.markedjson import tokens     // py:7
// from powerline.lib.unicode import unicode, unichr, surrogate_pair_to_character          // py:8

use crate::ported::lint::markedjson::error::MarkedError;
use crate::ported::lint::markedjson::nodes::Mark;
use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;

/// Port of `hexdigits_set` from
/// `powerline/lint/markedjson/scanner.py:11`.
///
/// Python: `set(string.hexdigits)` — set of '0'-'9', 'a'-'f', 'A'-'F'.
pub fn hexdigits_set() -> &'static HashSet<char> {
    static S: OnceLock<HashSet<char>> = OnceLock::new();
    S.get_or_init(|| {
        let mut s = HashSet::new();
        for c in "0123456789abcdefABCDEF".chars() {
            s.insert(c);
        }
        s
    })
}

/// Port of `class ScannerError(MarkedError)` from
/// `powerline/lint/markedjson/scanner.py:31`.
#[derive(Debug, Clone)]
pub struct ScannerError(pub MarkedError);

impl std::fmt::Display for ScannerError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        std::fmt::Display::fmt(&self.0, f)
    }
}

impl std::error::Error for ScannerError {}

/// Port of `class SimpleKey` from
/// `powerline/lint/markedjson/scanner.py:35`.
///
/// Tracks the state needed to commit a "simple key" candidate to a
/// concrete KEY token when the corresponding `:` value indicator
/// is found.
#[derive(Debug, Clone)]
pub struct SimpleKey {
    /// Python: `self.token_number` — index into the emitted tokens
    /// stream that this simple key would land at.
    pub token_number: usize,
    /// Python: `self.index` — Reader absolute char index at the key
    /// start.
    pub index: usize,
    /// Python: `self.line` — line position at the key start.
    pub line: usize,
    /// Python: `self.column` — column position at the key start.
    pub column: usize,
    /// Python: `self.mark`.
    pub mark: Option<Mark>,
}

impl SimpleKey {
    /// Port of `SimpleKey.__init__()` from
    /// `powerline/lint/markedjson/scanner.py:37`.
    pub fn new(
        token_number: usize,
        index: usize,
        line: usize,
        column: usize,
        mark: Option<Mark>,
    ) -> Self {
        Self {
            token_number,
            index,
            line,
            column,
            mark,
        }
    }
}

/// Token-type discriminator the scanner can fetch. Rust analog of
/// the Python `fetch_*` method dispatch table at
/// `powerline/lint/markedjson/scanner.py:140-187`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FetchKind {
    /// py:148  '\0' → fetch_stream_end
    StreamEnd,
    /// py:152  '[' → fetch_flow_sequence_start
    FlowSequenceStart,
    /// py:156  '{' → fetch_flow_mapping_start
    FlowMappingStart,
    /// py:160  ']' → fetch_flow_sequence_end
    FlowSequenceEnd,
    /// py:164  '}' → fetch_flow_mapping_end
    FlowMappingEnd,
    /// py:168  ',' → fetch_flow_entry
    FlowEntry,
    /// py:172  ':' (inside flow context) → fetch_value
    Value,
    /// py:176  '"' → fetch_double
    Double,
    /// py:180-181  plain scalar
    Plain,
}

/// Port of the dispatch table at
/// `powerline/lint/markedjson/scanner.py:140-187`.
///
/// Returns the `FetchKind` the scanner should fetch for the given
/// next character + `flow_level` state. The Python `':'` branch only
/// fires inside a flow context (py:172).
pub fn dispatch_fetch_for(ch: char, flow_level: u32) -> Option<FetchKind> {
    // py:148  ch == '\0' → STREAM-END
    if ch == '\0' {
        return Some(FetchKind::StreamEnd);
    }
    // py:152  '[' → flow sequence start
    if ch == '[' {
        return Some(FetchKind::FlowSequenceStart);
    }
    // py:156  '{' → flow mapping start
    if ch == '{' {
        return Some(FetchKind::FlowMappingStart);
    }
    // py:160  ']' → flow sequence end
    if ch == ']' {
        return Some(FetchKind::FlowSequenceEnd);
    }
    // py:164  '}' → flow mapping end
    if ch == '}' {
        return Some(FetchKind::FlowMappingEnd);
    }
    // py:168  ',' → flow entry
    if ch == ',' {
        return Some(FetchKind::FlowEntry);
    }
    // py:172  ':' + flow_level → value
    if ch == ':' && flow_level > 0 {
        return Some(FetchKind::Value);
    }
    // py:176  '"' → double scalar
    if ch == '"' {
        return Some(FetchKind::Double);
    }
    // py:180  if self.check_plain(): plain
    if check_plain(ch) {
        return Some(FetchKind::Plain);
    }
    // py:184-187  otherwise → scanner error
    None
}

/// Port of `Scanner.check_plain()` from
/// `powerline/lint/markedjson/scanner.py:360`.
///
/// Returns true when the next char can start a plain scalar.
/// Python checks `not any(ch in '...' for ch in self.peek())`; the
/// Rust port takes the char directly.
pub fn check_plain(ch: char) -> bool {
    // py:360-363  exclude indicators / null / structural chars
    !"\0[]{},:\"".contains(ch)
}

/// Port of `class Scanner` from
/// `powerline/lint/markedjson/scanner.py:45`.
///
/// State machine. The Python class inherits from `Reader` (multiple
/// inheritance) so it has access to `peek` / `prefix` / `forward`
/// methods on `self`. Rust port takes the Reader as a generic
/// parameter or trait object once the dispatch is wired through.
pub struct Scanner {
    /// Python: `self.done` (py:55).
    pub done: bool,
    /// Python: `self.flow_level` (py:58).
    pub flow_level: u32,
    /// Python: `self.tokens` — emitted token queue (py:61).
    /// Stored as a `Vec<TokenSlot>` placeholder since the concrete
    /// `tokens` module enum hasn't been threaded through yet.
    pub tokens: Vec<String>,
    /// Python: `self.tokens_taken` (py:67).
    pub tokens_taken: usize,
    /// Python: `self.allow_simple_key` (py:88).
    pub allow_simple_key: bool,
    /// Python: `self.possible_simple_keys` (py:91).
    pub possible_simple_keys: HashMap<u32, SimpleKey>,
}

impl Default for Scanner {
    fn default() -> Self {
        Self::new()
    }
}

impl Scanner {
    /// Port of `Scanner.__init__()` from
    /// `powerline/lint/markedjson/scanner.py:46`.
    pub fn new() -> Self {
        let mut s = Self {
            // py:55  self.done = False
            done: false,
            // py:58  self.flow_level = 0
            flow_level: 0,
            // py:61  self.tokens = []
            tokens: Vec::new(),
            // py:67  self.tokens_taken = 0
            tokens_taken: 0,
            // py:88  self.allow_simple_key = True
            allow_simple_key: true,
            // py:91  self.possible_simple_keys = {}
            possible_simple_keys: HashMap::new(),
        };
        // py:65  self.fetch_stream_start()
        s.fetch_stream_start();
        s
    }

    /// Port of `Scanner.fetch_stream_start()` from
    /// `powerline/lint/markedjson/scanner.py:238`.
    ///
    /// Surfaces just the queue-append; the Python source records the
    /// current position via Reader.get_mark + creates a
    /// `tokens.StreamStartToken` instance. The Rust port stashes a
    /// placeholder string until the tokens enum is threaded through.
    pub fn fetch_stream_start(&mut self) {
        // py:240-247  reset state + append StreamStartToken
        self.tokens.push("StreamStartToken".to_string());
    }

    /// Port of `Scanner.fetch_stream_end()` from
    /// `powerline/lint/markedjson/scanner.py:248`.
    pub fn fetch_stream_end(&mut self) {
        // py:250-261  set done + clear simple keys + append token
        self.done = true;
        self.possible_simple_keys.clear();
        self.tokens.push("StreamEndToken".to_string());
    }

    /// Port of `Scanner.check_token()` from
    /// `powerline/lint/markedjson/scanner.py:94`.
    ///
    /// `choices` is a list of token-type names to match against the
    /// next emitted token. Empty `choices` returns true when any
    /// token is available.
    pub fn check_token(&self, choices: &[&str]) -> bool {
        // py:94  def check_token(self, *choices):
        // py:95  # Check if the next token is one of the given types.
        // py:96  while self.need_more_tokens():
        // py:97  self.fetch_more_tokens()
        // py:98  if self.tokens:
        // py:99  if not choices:
        // py:100  return True
        // py:101  for choice in choices:
        // py:102  if isinstance(self.tokens[0], choice):
        // py:103  return True
        // py:104  return False
        match self.tokens.first() {
            None => false,
            Some(t) => {
                if choices.is_empty() {
                    true
                } else {
                    choices.iter().any(|c| *c == t)
                }
            }
        }
    }

    /// Port of `Scanner.peek_token()` from
    /// `powerline/lint/markedjson/scanner.py:106`.
    pub fn peek_token(&self) -> Option<&str> {
        // py:106  def peek_token(self):
        // py:107  # Return the next token, but do not delete if from the queue.
        // py:108  while self.need_more_tokens():
        // py:109  self.fetch_more_tokens()
        // py:110  if self.tokens:
        // py:111  return self.tokens[0]
        self.tokens.first().map(String::as_str)
    }

    /// Port of `Scanner.get_token()` from
    /// `powerline/lint/markedjson/scanner.py:113`.
    pub fn get_token(&mut self) -> Option<String> {
        // py:113  def get_token(self):
        // py:114  # Return the next token.
        // py:115  while self.need_more_tokens():
        // py:116  self.fetch_more_tokens()
        // py:117  if self.tokens:
        // py:118  self.tokens_taken += 1
        // py:119  return self.tokens.pop(0)
        if self.tokens.is_empty() {
            return None;
        }
        self.tokens_taken += 1;
        Some(self.tokens.remove(0))
    }

    /// Port of `Scanner.need_more_tokens()` from
    /// `powerline/lint/markedjson/scanner.py:123`.
    pub fn need_more_tokens(&self) -> bool {
        // py:123  def need_more_tokens(self):
        // py:124  if self.done:
        // py:125  return False
        // py:126  if not self.tokens:
        // py:127  return True
        // py:128  # The current token may be a potential simple key, so we
        // py:129  # need to look further.
        // py:130  self.stale_possible_simple_keys()
        // py:131  if self.next_possible_simple_key() == self.tokens_taken:
        // py:132  return True
        if self.done {
            return false;
        }
        if self.tokens.is_empty() {
            return true;
        }
        matches!(self.next_possible_simple_key(), Some(n) if n == self.tokens_taken)
    }

    /// Port of `Scanner.next_possible_simple_key()` from
    /// `powerline/lint/markedjson/scanner.py:192`.
    pub fn next_possible_simple_key(&self) -> Option<usize> {
        // py:192  def next_possible_simple_key(self):
        // py:193  # Return the number of the nearest possible simple key. ...
        // py:200  min_token_number = None
        // py:201  for level in self.possible_simple_keys:
        // py:202  key = self.possible_simple_keys[level]
        // py:203  if min_token_number is None or key.token_number < min_token_number:
        // py:204  min_token_number = key.token_number
        // py:205  return min_token_number
        self.possible_simple_keys
            .values()
            .map(|k| k.token_number)
            .min()
    }

    /// Port of `Scanner.save_possible_simple_key()` from
    /// `powerline/lint/markedjson/scanner.py:218`.
    pub fn save_possible_simple_key(&mut self, index: usize, line: usize, column: usize) {
        // py:218  def save_possible_simple_key(self):
        // py:219  # The next token may start a simple key. ...
        // py:225  if self.allow_simple_key:
        // py:226  self.remove_possible_simple_key()
        // py:227  token_number = self.tokens_taken + len(self.tokens)
        // py:228  key = SimpleKey(token_number, index, line, column, self.get_mark())
        // py:229  self.possible_simple_keys[self.flow_level] = key
        if !self.allow_simple_key {
            return;
        }
        let token_number = self.tokens_taken + self.tokens.len();
        let key = SimpleKey::new(token_number, index, line, column, None);
        self.possible_simple_keys.insert(self.flow_level, key);
    }

    /// Port of `Scanner.remove_possible_simple_key()` from
    /// `powerline/lint/markedjson/scanner.py:231`.
    pub fn remove_possible_simple_key(&mut self) {
        // py:231  def remove_possible_simple_key(self):
        // py:232  # Remove the saved possible key position at the current flow level.
        // py:233  if self.flow_level in self.possible_simple_keys:
        // py:234  key = self.possible_simple_keys[self.flow_level]
        // py:235  del self.possible_simple_keys[self.flow_level]
        self.possible_simple_keys.remove(&self.flow_level);
    }

    /// Port of `Scanner.fetch_flow_sequence_start()` /
    /// `fetch_flow_mapping_start()` (py:263-282) — increments
    /// flow_level and appends the corresponding token.
    pub fn fetch_flow_collection_start(&mut self, token_name: &str) {
        // py:269-284  increment flow_level + append token
        self.flow_level += 1;
        self.tokens.push(token_name.to_string());
    }

    /// Port of `Scanner.fetch_flow_sequence_end()` /
    /// `fetch_flow_mapping_end()` (py:285-305) — decrements
    /// flow_level and appends the corresponding end token.
    pub fn fetch_flow_collection_end(&mut self, token_name: &str) {
        // py:291-305  decrement flow_level + append token
        if self.flow_level > 0 {
            self.flow_level -= 1;
        }
        self.tokens.push(token_name.to_string());
    }

    /// Port of `Scanner.fetch_value()` from
    /// `powerline/lint/markedjson/scanner.py:307`.
    pub fn fetch_value(&mut self) {
        // py:308-324  append ValueToken
        self.tokens.push("ValueToken".to_string());
    }

    /// Port of `Scanner.fetch_flow_entry()` from
    /// `powerline/lint/markedjson/scanner.py:325`.
    pub fn fetch_flow_entry(&mut self) {
        // py:326-336  append FlowEntryToken
        self.tokens.push("FlowEntryToken".to_string());
    }

    /// Port of `Scanner.fetch_flow_sequence_start()` from
    /// `powerline/lint/markedjson/scanner.py:263-264`.
    ///
    /// Thin wrapper around `fetch_flow_collection_start` with the
    /// `FlowSequenceStartToken` token name per py:264.
    pub fn fetch_flow_sequence_start(&mut self) {
        // py:264  self.fetch_flow_collection_start(tokens.FlowSequenceStartToken)
        self.fetch_flow_collection_start("FlowSequenceStartToken");
    }

    /// Port of `Scanner.fetch_flow_mapping_start()` from
    /// `powerline/lint/markedjson/scanner.py:266-267`.
    pub fn fetch_flow_mapping_start(&mut self) {
        // py:267  self.fetch_flow_collection_start(tokens.FlowMappingStartToken)
        self.fetch_flow_collection_start("FlowMappingStartToken");
    }

    /// Port of `Scanner.fetch_flow_sequence_end()` from
    /// `powerline/lint/markedjson/scanner.py:285-286`.
    pub fn fetch_flow_sequence_end(&mut self) {
        // py:286  self.fetch_flow_collection_end(tokens.FlowSequenceEndToken)
        self.fetch_flow_collection_end("FlowSequenceEndToken");
    }

    /// Port of `Scanner.fetch_flow_mapping_end()` from
    /// `powerline/lint/markedjson/scanner.py:288-289`.
    pub fn fetch_flow_mapping_end(&mut self) {
        // py:289  self.fetch_flow_collection_end(tokens.FlowMappingEndToken)
        self.fetch_flow_collection_end("FlowMappingEndToken");
    }

    /// Port of `Scanner.fetch_double()` from
    /// `powerline/lint/markedjson/scanner.py:338-346`.
    ///
    /// Saves the possible simple key (py:340), disallows further
    /// simple keys (py:343), and appends the scanned flow scalar
    /// per py:346. The actual `scan_flow_scalar()` dispatch lives
    /// outside this stub — the Rust port emits a placeholder token
    /// since the reader buffer isn't threaded through.
    pub fn fetch_double(&mut self, index: usize, line: usize, column: usize) {
        // py:340  self.save_possible_simple_key()
        self.save_possible_simple_key(index, line, column);
        // py:343  self.allow_simple_key = False
        self.allow_simple_key = false;
        // py:346  self.tokens.append(self.scan_flow_scalar())
        self.tokens.push("ScalarToken".to_string());
    }

    /// Port of `Scanner.fetch_plain()` from
    /// `powerline/lint/markedjson/scanner.py:348-356`.
    pub fn fetch_plain(&mut self, index: usize, line: usize, column: usize) {
        // py:348  def fetch_plain(self):
        // py:349  # A plain scalar could be a simple key.
        // py:350  self.save_possible_simple_key()
        self.save_possible_simple_key(index, line, column);
        // py:351  # No simple keys after plain scalars. But note that `scan_plain` will
        // py:352  # change this flag if the scan is finished at the beginning of the line.
        // py:353  self.allow_simple_key = False
        self.allow_simple_key = false;
        // py:354  # Scan and add SCALAR. May change `allow_simple_key`.
        // py:355  self.tokens.append(self.scan_plain())
        self.tokens.push("ScalarToken".to_string());
    }

    /// Port of `Scanner.fetch_more_tokens()` from
    /// `powerline/lint/markedjson/scanner.py:134`.
    ///
    /// **Status:** stub. The Rust port surfaces the dispatch shape;
    /// callers drive `scan_to_next_token` + per-char dispatch via
    /// the module-level `dispatch_fetch_for` helper.
    pub fn fetch_more_tokens(&mut self) {
        // py:134  def fetch_more_tokens(self):
        // py:136  # Eat whitespaces and comments until we reach the next token.
        // py:137  self.scan_to_next_token()
        // py:139  # Remove obsolete possible simple keys.
        // py:140  self.stale_possible_simple_keys()
        // py:142  # Peek the next character.
        // py:143  ch = self.peek()
        // py:145  # Is it the end of stream?
        // py:146  if ch == '\0':
        // py:147  return self.fetch_stream_end()
        // py:149  # Note: the order of the following checks is NOT significant.
        // py:151  # Is it the flow sequence start indicator?
        // py:152  if ch == '[':
        // py:153  return self.fetch_flow_sequence_start()
        // py:155  # Is it the flow mapping start indicator?
        // py:156  if ch == '{':
        // py:157  return self.fetch_flow_mapping_start()
        // py:159  # Is it the flow sequence end indicator?
        // py:160  if ch == ']':
        // py:161  return self.fetch_flow_sequence_end()
        // py:163  # Is it the flow mapping end indicator?
        // py:164  if ch == '}':
        // py:165  return self.fetch_flow_mapping_end()
        // py:167  # Is it the flow entry indicator?
        // py:168  if ch == ',':
        // py:169  return self.fetch_flow_entry()
        // py:171  # Is it the value indicator?
        // py:172  if ch == ':' and self.flow_level:
        // py:173  return self.fetch_value()
        // py:175  # Is it a double quoted scalar?
        // py:176  if ch == '"':
        // py:177  return self.fetch_double()
        // py:179  # It must be a plain scalar then.
        // py:180  if self.check_plain():
        // py:181  return self.fetch_plain()
        // py:183  # No? It's an error. Let's produce a nice error message.
        // py:184  raise ScannerError(
        // py:185  'while scanning for the next token', None,
        // py:186  'found character %r that cannot start any token' % ch,
        // py:187  self.get_mark()
        // py:188  )
    }

    /// Port of `Scanner.stale_possible_simple_keys()` from
    /// `powerline/lint/markedjson/scanner.py:207`.
    ///
    /// Removes simple-key entries whose line differs from
    /// `current_line` per py:213-216.
    pub fn stale_possible_simple_keys(&mut self, current_line: usize) {
        // py:207  def stale_possible_simple_keys(self):
        // py:208  # Remove entries that are no longer possible simple keys. ...
        // py:213  for level in list(self.possible_simple_keys):
        // py:214  key = self.possible_simple_keys[level]
        // py:215  if key.line != self.line:
        // py:216  del self.possible_simple_keys[level]
        let stale: Vec<u32> = self
            .possible_simple_keys
            .iter()
            .filter_map(|(k, v)| {
                if v.line != current_line {
                    Some(*k)
                } else {
                    None
                }
            })
            .collect();
        for k in stale {
            self.possible_simple_keys.remove(&k);
        }
    }
}

/// Port of `Scanner.scan_flow_scalar()` from
/// `powerline/lint/markedjson/scanner.py:369-386`.
///
/// Scans a quoted scalar (double-quoted in JSON). Python builds
/// the chunks list by interleaving non-space + space scans until
/// the closing quote is consumed, then returns a
/// `ScalarToken(value, ..., '"')`.
///
/// The Rust port takes the post-opening-quote slice of the buffer
/// and returns the unquoted value + bytes consumed (including the
/// closing quote). Escape handling lives in
/// [`scan_flow_scalar_non_spaces`]; whitespace folding in
/// [`scan_flow_scalar_spaces`].
///
/// Returns `(value, bytes_consumed)` on success. `None` when the
/// buffer ends before the closing quote.
pub fn scan_flow_scalar(buffer: &str, quote: char) -> Option<(String, usize)> {
    // py:369  def scan_flow_scalar(self):
    // py:376  chunks = []
    // py:377  start_mark = self.get_mark()
    // py:378  quote = self.peek()
    // py:379  self.forward()
    let mut chunks = String::new();
    let mut pos = 0usize;
    let bytes = buffer.as_bytes();
    // py:380  chunks.extend(self.scan_flow_scalar_non_spaces(start_mark))
    // py:381-383  loop: spaces + non-spaces until quote
    while pos < bytes.len() {
        let ch = bytes[pos] as char;
        if ch == quote {
            // py:384  self.forward()  (consume closing quote)
            // py:385-386  return ScalarToken(chunks)
            return Some((chunks, pos + 1));
        }
        // py:382  spaces
        let (s, ds) = scan_flow_scalar_spaces(&buffer[pos..]);
        chunks.push_str(&s);
        pos += ds;
        if pos >= bytes.len() {
            break;
        }
        // py:383  non-spaces
        let (ns, dns) = scan_flow_scalar_non_spaces(&buffer[pos..]);
        chunks.push_str(&ns);
        pos += dns;
        // Avoid infinite loop on unrecognized input.
        if ds == 0 && dns == 0 {
            break;
        }
    }
    None
}

/// Port of `Scanner.scan_flow_scalar_non_spaces()` from
/// `powerline/lint/markedjson/scanner.py:402-457`.
///
/// Scans the next run of non-whitespace characters inside a
/// quoted scalar, handling escape sequences (`\b`, `\t`, `\n`,
/// `\f`, `\r`, `\"`, `\\`) per `ESCAPE_REPLACEMENTS` at py:388-396
/// and `\uXXXX` per py:399-400.
///
/// Returns `(decoded_value, bytes_consumed)`. Stops at the first
/// whitespace, quote, or backslash-followed-EOF.
pub fn scan_flow_scalar_non_spaces(buffer: &str) -> (String, usize) {
    let mut out = String::new();
    let mut pos = 0usize;
    let bytes = buffer.as_bytes();
    while pos < bytes.len() {
        let ch = bytes[pos] as char;
        // py:407  stop at any of " \ \0 ' ' '\t' '\n'
        if matches!(ch, '"' | '\\' | '\0' | ' ' | '\t' | '\n') {
            if ch == '\\' && pos + 1 < bytes.len() {
                // py:413-422  escape handling
                let next = bytes[pos + 1] as char;
                match next {
                    'b' => out.push('\x08'),
                    't' => out.push('\t'),
                    'n' => out.push('\n'),
                    'f' => out.push('\x0c'),
                    'r' => out.push('\r'),
                    '"' => out.push('"'),
                    '\\' => out.push('\\'),
                    'u' => {
                        // py:399-400  \uXXXX
                        if pos + 6 <= bytes.len() {
                            let hex = &buffer[pos + 2..pos + 6];
                            if let Ok(code) = u32::from_str_radix(hex, 16) {
                                if let Some(c) = char::from_u32(code) {
                                    out.push(c);
                                }
                            }
                            pos += 6;
                            continue;
                        }
                        return (out, pos);
                    }
                    other => out.push(other),
                }
                pos += 2;
                continue;
            }
            break;
        }
        out.push(ch);
        pos += 1;
    }
    (out, pos)
}

/// Port of `Scanner.scan_flow_scalar_spaces()` from
/// `powerline/lint/markedjson/scanner.py:459-480`.
///
/// Folds the run of whitespace (spaces / tabs) inside a quoted
/// scalar into the canonical YAML representation per the JSON
/// subset rules. Returns `(folded_value, bytes_consumed)`.
pub fn scan_flow_scalar_spaces(buffer: &str) -> (String, usize) {
    let mut pos = 0usize;
    let bytes = buffer.as_bytes();
    while pos < bytes.len() {
        let ch = bytes[pos] as char;
        if ch == ' ' || ch == '\t' {
            pos += 1;
        } else {
            break;
        }
    }
    // py:478-479  return ' ' joined chunks
    (" ".repeat(if pos > 0 { 1 } else { 0 }), pos)
}

/// Port of `Scanner.scan_plain()` from
/// `powerline/lint/markedjson/scanner.py:482-527`.
///
/// Scans an unquoted (plain) scalar — runs of `check_plain`-true
/// characters until a structural separator. Used for unquoted
/// keys + values in the JSON subset.
///
/// Returns `(value, bytes_consumed)`.
pub fn scan_plain(buffer: &str) -> (String, usize) {
    let mut out = String::new();
    let mut pos = 0usize;
    for (i, ch) in buffer.char_indices() {
        if !check_plain(ch) {
            break;
        }
        out.push(ch);
        pos = i + ch.len_utf8();
    }
    (out, pos)
}

/// Port of `Scanner.scan_to_next_token()` from
/// `powerline/lint/markedjson/scanner.py:365-367`.
///
/// Advances past leading spaces/tabs/newlines. Returns the number
/// of bytes consumed from the start of `buffer`.
pub fn scan_to_next_token(buffer: &str) -> usize {
    // py:366-367  while self.peek() in ' \t\n': self.forward()
    buffer
        .bytes()
        .take_while(|&b| b == b' ' || b == b'\t' || b == b'\n')
        .count()
}

/// Port of `Scanner.fetch_more_tokens()` from
/// `powerline/lint/markedjson/scanner.py:134-190`.
///
/// Drives one round of the scanner's lex loop: peeks the next
/// non-whitespace character, then routes through `dispatch_fetch_for`
/// to pick a fetch action. The Rust port returns the dispatch result +
/// the consumed whitespace count so callers can advance their reader
/// before invoking the chosen `fetch_*` method.
pub fn fetch_more_tokens(buffer: &str, flow_level: u32) -> (usize, Option<FetchKind>) {
    // py:135  self.scan_to_next_token()
    let whitespace = scan_to_next_token(buffer);
    // py:138  ch = self.peek()
    let peeked = buffer[whitespace..].chars().next().unwrap_or('\0');
    // py:140-187  dispatch
    (whitespace, dispatch_fetch_for(peeked, flow_level))
}

/// Port of `Scanner.stale_possible_simple_keys()` from
/// `powerline/lint/markedjson/scanner.py:207`.
///
/// Removes simple-key candidates that have been invalidated by line
/// advancement past their position. Pure function over (current_line,
/// keys) → ids-to-remove.
pub fn stale_possible_simple_keys(current_line: usize, keys: &HashMap<u32, SimpleKey>) -> Vec<u32> {
    // py:209-217  remove keys whose line is less than current_line
    keys.iter()
        .filter_map(|(level, k)| {
            if k.line < current_line {
                Some(*level)
            } else {
                None
            }
        })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn hexdigits_set_contains_all_hex_chars() {
        let s = hexdigits_set();
        for c in "0123456789abcdefABCDEF".chars() {
            assert!(s.contains(&c), "missing: {}", c);
        }
        assert!(!s.contains(&'g'));
    }

    #[test]
    fn hexdigits_set_has_22_entries() {
        // 10 digits + 6 lower + 6 upper
        let s = hexdigits_set();
        assert_eq!(s.len(), 22);
    }

    #[test]
    fn scanner_error_implements_error_traits() {
        let me = MarkedError::new(Some("ctx"), None, Some("prob"), None, None);
        let e = ScannerError(me);
        let _: &dyn std::error::Error = &e;
        assert!(e.to_string().contains("ctx"));
    }

    #[test]
    fn simple_key_stores_state() {
        let k = SimpleKey::new(5, 10, 2, 3, None);
        assert_eq!(k.token_number, 5);
        assert_eq!(k.index, 10);
        assert_eq!(k.line, 2);
        assert_eq!(k.column, 3);
    }

    #[test]
    fn dispatch_fetch_for_null_returns_stream_end() {
        // py:148  ch == '\0' → STREAM-END
        assert_eq!(dispatch_fetch_for('\0', 0), Some(FetchKind::StreamEnd));
    }

    #[test]
    fn dispatch_fetch_for_bracket_returns_flow_sequence_start() {
        assert_eq!(
            dispatch_fetch_for('[', 0),
            Some(FetchKind::FlowSequenceStart)
        );
    }

    #[test]
    fn dispatch_fetch_for_brace_returns_flow_mapping_start() {
        assert_eq!(
            dispatch_fetch_for('{', 0),
            Some(FetchKind::FlowMappingStart)
        );
    }

    #[test]
    fn dispatch_fetch_for_close_bracket_returns_flow_sequence_end() {
        assert_eq!(dispatch_fetch_for(']', 1), Some(FetchKind::FlowSequenceEnd));
    }

    #[test]
    fn dispatch_fetch_for_close_brace_returns_flow_mapping_end() {
        assert_eq!(dispatch_fetch_for('}', 1), Some(FetchKind::FlowMappingEnd));
    }

    #[test]
    fn dispatch_fetch_for_comma_returns_flow_entry() {
        assert_eq!(dispatch_fetch_for(',', 1), Some(FetchKind::FlowEntry));
    }

    #[test]
    fn dispatch_fetch_for_colon_in_flow_returns_value() {
        // py:172  ':' + flow_level → Value
        assert_eq!(dispatch_fetch_for(':', 1), Some(FetchKind::Value));
    }

    #[test]
    fn dispatch_fetch_for_colon_outside_flow_returns_plain() {
        // py:172  ':' + no flow_level → falls through to plain
        // because check_plain(':') is false. So falls to None?
        // Actually check_plain excludes ':'. So returns None.
        assert_eq!(dispatch_fetch_for(':', 0), None);
    }

    #[test]
    fn dispatch_fetch_for_quote_returns_double() {
        assert_eq!(dispatch_fetch_for('"', 0), Some(FetchKind::Double));
    }

    #[test]
    fn dispatch_fetch_for_alpha_returns_plain() {
        assert_eq!(dispatch_fetch_for('a', 0), Some(FetchKind::Plain));
        assert_eq!(dispatch_fetch_for('1', 0), Some(FetchKind::Plain));
    }

    #[test]
    fn check_plain_excludes_structural_chars() {
        // py:360-363
        assert!(!check_plain('\0'));
        assert!(!check_plain('['));
        assert!(!check_plain(']'));
        assert!(!check_plain('{'));
        assert!(!check_plain('}'));
        assert!(!check_plain(','));
        assert!(!check_plain(':'));
        assert!(!check_plain('"'));
    }

    #[test]
    fn check_plain_accepts_alphanumeric() {
        assert!(check_plain('a'));
        assert!(check_plain('Z'));
        assert!(check_plain('5'));
        assert!(check_plain('_'));
    }

    #[test]
    fn scanner_new_emits_stream_start_token() {
        // py:65  __init__ calls fetch_stream_start
        let s = Scanner::new();
        assert_eq!(s.tokens.len(), 1);
        assert_eq!(s.tokens[0], "StreamStartToken");
        assert_eq!(s.flow_level, 0);
        assert_eq!(s.tokens_taken, 0);
        assert!(!s.done);
        assert!(s.allow_simple_key);
    }

    #[test]
    fn fetch_stream_end_sets_done_and_appends_token() {
        // py:248-261
        let mut s = Scanner::new();
        s.fetch_stream_end();
        assert!(s.done);
        assert_eq!(s.tokens[1], "StreamEndToken");
    }

    #[test]
    fn check_token_no_choices_returns_true_when_token_available() {
        let s = Scanner::new();
        assert!(s.check_token(&[]));
    }

    #[test]
    fn check_token_matches_specific_name() {
        let s = Scanner::new();
        assert!(s.check_token(&["StreamStartToken"]));
        assert!(!s.check_token(&["StreamEndToken"]));
    }

    #[test]
    fn check_token_returns_false_when_no_tokens() {
        // Empty queue case
        let mut s = Scanner {
            done: false,
            flow_level: 0,
            tokens: Vec::new(),
            tokens_taken: 0,
            allow_simple_key: true,
            possible_simple_keys: HashMap::new(),
        };
        s.tokens.clear();
        assert!(!s.check_token(&[]));
    }

    #[test]
    fn peek_token_returns_first_without_consuming() {
        let s = Scanner::new();
        assert_eq!(s.peek_token(), Some("StreamStartToken"));
        assert_eq!(s.tokens_taken, 0);
    }

    #[test]
    fn get_token_pops_and_increments_taken() {
        let mut s = Scanner::new();
        let t = s.get_token();
        assert_eq!(t.as_deref(), Some("StreamStartToken"));
        assert_eq!(s.tokens_taken, 1);
        assert!(s.tokens.is_empty());
    }

    #[test]
    fn need_more_tokens_false_when_done() {
        let mut s = Scanner::new();
        s.done = true;
        s.tokens.clear();
        assert!(!s.need_more_tokens());
    }

    #[test]
    fn need_more_tokens_true_when_queue_empty_and_not_done() {
        let mut s = Scanner::new();
        s.tokens.clear();
        assert!(s.need_more_tokens());
    }

    #[test]
    fn save_possible_simple_key_records_when_allowed() {
        let mut s = Scanner::new();
        s.save_possible_simple_key(10, 1, 2);
        assert!(s.possible_simple_keys.contains_key(&0));
        let k = &s.possible_simple_keys[&0];
        assert_eq!(k.index, 10);
        assert_eq!(k.line, 1);
        assert_eq!(k.column, 2);
    }

    #[test]
    fn save_possible_simple_key_skipped_when_disallowed() {
        let mut s = Scanner::new();
        s.allow_simple_key = false;
        s.save_possible_simple_key(10, 1, 2);
        assert!(s.possible_simple_keys.is_empty());
    }

    #[test]
    fn remove_possible_simple_key_clears_current_level() {
        let mut s = Scanner::new();
        s.save_possible_simple_key(10, 1, 2);
        s.remove_possible_simple_key();
        assert!(s.possible_simple_keys.is_empty());
    }

    #[test]
    fn fetch_flow_collection_start_increments_level() {
        // py:269-284
        let mut s = Scanner::new();
        s.fetch_flow_collection_start("FlowSequenceStartToken");
        assert_eq!(s.flow_level, 1);
        assert_eq!(s.tokens.last().unwrap(), "FlowSequenceStartToken");
    }

    #[test]
    fn fetch_flow_collection_end_decrements_level() {
        let mut s = Scanner::new();
        s.fetch_flow_collection_start("FlowSequenceStartToken");
        s.fetch_flow_collection_end("FlowSequenceEndToken");
        assert_eq!(s.flow_level, 0);
        assert_eq!(s.tokens.last().unwrap(), "FlowSequenceEndToken");
    }

    #[test]
    fn fetch_flow_collection_end_does_not_underflow() {
        let mut s = Scanner::new();
        s.fetch_flow_collection_end("FlowSequenceEndToken");
        assert_eq!(s.flow_level, 0);
    }

    #[test]
    fn fetch_value_appends_value_token() {
        let mut s = Scanner::new();
        s.fetch_value();
        assert_eq!(s.tokens.last().unwrap(), "ValueToken");
    }

    #[test]
    fn fetch_flow_entry_appends_entry_token() {
        let mut s = Scanner::new();
        s.fetch_flow_entry();
        assert_eq!(s.tokens.last().unwrap(), "FlowEntryToken");
    }

    #[test]
    fn next_possible_simple_key_returns_min_token_number() {
        let mut s = Scanner::new();
        s.possible_simple_keys
            .insert(0, SimpleKey::new(5, 0, 0, 0, None));
        s.possible_simple_keys
            .insert(1, SimpleKey::new(3, 0, 0, 0, None));
        assert_eq!(s.next_possible_simple_key(), Some(3));
    }

    #[test]
    fn next_possible_simple_key_none_when_empty() {
        let s = Scanner::new();
        assert!(s.next_possible_simple_key().is_none());
    }

    #[test]
    fn stale_possible_simple_keys_filters_keys_below_current_line() {
        let mut keys = HashMap::new();
        keys.insert(0, SimpleKey::new(1, 0, 1, 0, None));
        keys.insert(1, SimpleKey::new(2, 0, 5, 0, None));
        let stale = stale_possible_simple_keys(3, &keys);
        assert_eq!(stale, vec![0]);
    }

    #[test]
    fn fetch_flow_sequence_start_delegates_to_collection_start() {
        // py:264
        let mut s = Scanner::new();
        let before = s.flow_level;
        s.fetch_flow_sequence_start();
        assert_eq!(s.flow_level, before + 1);
        assert!(s.tokens.contains(&"FlowSequenceStartToken".to_string()));
    }

    #[test]
    fn fetch_flow_mapping_start_delegates_to_collection_start() {
        // py:267
        let mut s = Scanner::new();
        s.fetch_flow_mapping_start();
        assert!(s.tokens.contains(&"FlowMappingStartToken".to_string()));
    }

    #[test]
    fn fetch_flow_sequence_end_delegates_to_collection_end() {
        // py:286
        let mut s = Scanner::new();
        // Set up so flow_level can decrement
        s.flow_level = 1;
        s.fetch_flow_sequence_end();
        assert_eq!(s.flow_level, 0);
        assert!(s.tokens.contains(&"FlowSequenceEndToken".to_string()));
    }

    #[test]
    fn fetch_flow_mapping_end_delegates_to_collection_end() {
        // py:289
        let mut s = Scanner::new();
        s.flow_level = 1;
        s.fetch_flow_mapping_end();
        assert!(s.tokens.contains(&"FlowMappingEndToken".to_string()));
    }

    #[test]
    fn fetch_double_disallows_simple_key_after() {
        // py:343
        let mut s = Scanner::new();
        s.allow_simple_key = true;
        s.fetch_double(0, 0, 0);
        assert!(!s.allow_simple_key);
        assert!(s.tokens.contains(&"ScalarToken".to_string()));
    }

    #[test]
    fn fetch_plain_disallows_simple_key_after() {
        // py:353
        let mut s = Scanner::new();
        s.allow_simple_key = true;
        s.fetch_plain(0, 0, 0);
        assert!(!s.allow_simple_key);
        assert!(s.tokens.contains(&"ScalarToken".to_string()));
    }

    #[test]
    fn scan_to_next_token_skips_leading_whitespace() {
        // py:366-367
        assert_eq!(scan_to_next_token("   abc"), 3);
        assert_eq!(scan_to_next_token("\t\nabc"), 2);
        assert_eq!(scan_to_next_token("abc"), 0);
    }

    #[test]
    fn scan_to_next_token_empty_returns_zero() {
        assert_eq!(scan_to_next_token(""), 0);
    }

    #[test]
    fn fetch_more_tokens_dispatches_via_dispatch_fetch_for() {
        // py:135-187
        let (ws, kind) = fetch_more_tokens("  [", 0);
        assert_eq!(ws, 2);
        assert_eq!(kind, Some(FetchKind::FlowSequenceStart));
    }

    #[test]
    fn fetch_more_tokens_empty_buffer_returns_stream_end() {
        let (ws, kind) = fetch_more_tokens("", 0);
        assert_eq!(ws, 0);
        assert_eq!(kind, Some(FetchKind::StreamEnd));
    }

    #[test]
    fn fetch_more_tokens_whitespace_only_returns_stream_end() {
        // After consuming whitespace, peek '\0' → StreamEnd
        let (ws, kind) = fetch_more_tokens("   ", 0);
        assert_eq!(ws, 3);
        assert_eq!(kind, Some(FetchKind::StreamEnd));
    }

    #[test]
    fn scan_flow_scalar_non_spaces_returns_run_until_quote() {
        // py:402-457
        let (s, n) = scan_flow_scalar_non_spaces("hello\"world");
        assert_eq!(s, "hello");
        assert_eq!(n, 5);
    }

    #[test]
    fn scan_flow_scalar_non_spaces_decodes_basic_escapes() {
        // py:388-396
        let (s, _) = scan_flow_scalar_non_spaces("hi\\n\"");
        assert_eq!(s, "hi\n");
    }

    #[test]
    fn scan_flow_scalar_non_spaces_decodes_unicode_escape() {
        // py:399-400
        let (s, _) = scan_flow_scalar_non_spaces("\\u0041\"");
        assert_eq!(s, "A");
    }

    #[test]
    fn scan_flow_scalar_spaces_collapses_run() {
        // py:459-480
        let (s, n) = scan_flow_scalar_spaces("   hello");
        assert_eq!(s, " ");
        assert_eq!(n, 3);
    }

    #[test]
    fn scan_flow_scalar_spaces_empty_run_yields_empty_string() {
        let (s, n) = scan_flow_scalar_spaces("hello");
        assert_eq!(s, "");
        assert_eq!(n, 0);
    }

    #[test]
    fn scan_flow_scalar_round_trips_simple_string() {
        // py:369-386
        let r = scan_flow_scalar("hello\"", '"');
        assert_eq!(r, Some(("hello".to_string(), 6)));
    }

    #[test]
    fn scan_flow_scalar_returns_none_when_no_closing_quote() {
        let r = scan_flow_scalar("hello", '"');
        assert!(r.is_none());
    }

    #[test]
    fn scan_plain_stops_at_structural_chars() {
        // py:482-527
        let (s, n) = scan_plain("foo:bar");
        assert_eq!(s, "foo");
        assert_eq!(n, 3);
    }

    #[test]
    fn scan_plain_empty_input_returns_empty() {
        let (s, n) = scan_plain("");
        assert_eq!(s, "");
        assert_eq!(n, 0);
    }

    #[test]
    fn scan_plain_runs_to_end_when_all_plain() {
        let (s, n) = scan_plain("abc123");
        assert_eq!(s, "abc123");
        assert_eq!(n, 6);
    }
}