Struct text_scanner::Scanner

pub struct Scanner<'text> { /* private fields */ }

A Scanner is a UTF-8 char text scanner, implementing various methods for scanning a string slice, as well as backtracking capabilities, which can be used to implement lexers for tokenizing text or code. It is essentially just a fancy wrapper around CharRanges.

Implementations

impl<'text> Scanner<'text>

pub fn new(text: &'text str) -> Self

Constructs a new Scanner with text.

pub fn text(&self) -> &'text str

Returns the text the scanner was constructed with.

Note: This has the same lifetime as the original text, so the scanner can continue to be used while this exists.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.next(), Ok((0..1, 'H')));
assert_eq!(scanner.next(), Ok((1..2, 'e')));

assert_eq!(scanner.text(), "Hello World");
assert_eq!(scanner.remaining_text(), "llo World");

pub fn remaining_text(&self) -> &'text str

Returns the remaining text of the scanner, i.e. the text() after cursor_pos(), in other words self.text()[self.cursor_pos()..].

Note: This has the same lifetime as the original text, so the scanner can continue to be used while this exists.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.text(), "Hello World");
assert_eq!(scanner.remaining_text(), "Hello World");

assert_eq!(scanner.next(), Ok((0..1, 'H')));
assert_eq!(scanner.next(), Ok((1..2, 'e')));

assert_eq!(scanner.text(), "Hello World");
assert_eq!(scanner.remaining_text(), "llo World");

pub fn has_remaining_text(&self) -> bool

Returns true if remaining_text() has text, i.e. if it is not empty.

Example

let mut scanner = Scanner::new("Foo");

assert_eq!(scanner.remaining_text(), "Foo");
assert_eq!(scanner.has_remaining_text(), true);

assert_eq!(scanner.next(), Ok((0..1, 'F')));
assert_eq!(scanner.next(), Ok((1..2, 'o')));
assert_eq!(scanner.next(), Ok((2..3, 'o')));

assert_eq!(scanner.remaining_text(), "");
assert_eq!(scanner.has_remaining_text(), false);

pub fn ranged_text(&self, range: Range<usize>) -> ScannerItem<&'text str>

pub fn cursor_pos(&self) -> usize

Returns the current cursor position of the scanner, i.e. the byte offset into text().

pub fn set_cursor_pos(&mut self, pos: usize) -> usize

Replaces the current cursor position with pos, while returning the old cursor position.

Panics

If pos is not at a valid UTF-8 sequence boundary, then the next operation using the cursor position will panic.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.next(), Ok((0..1, 'H')));

let backtrack = scanner.cursor_pos();

assert_eq!(scanner.next(), Ok((1..2, 'e')));
assert_eq!(scanner.next(), Ok((2..3, 'l')));
assert_eq!(scanner.next(), Ok((3..4, 'l')));

scanner.set_cursor_pos(backtrack);

assert_eq!(scanner.next(), Ok((1..2, 'e')));
assert_eq!(scanner.next(), Ok((2..3, 'l')));
assert_eq!(scanner.next(), Ok((3..4, 'l')));

pub fn reset(&mut self) -> usize

Resets the cursor position to the start, while returning the old cursor position.

Example

let old_pos = scanner.reset();
// same as
let old_pos = scanner.set_cursor_pos(0);

pub fn next(&mut self) -> ScannerResult<'text, char>

Advances the scanner cursor and returns the next char and its Range, if any.

Example

let mut scanner = Scanner::new("Hello");

assert_eq!(scanner.next(), Ok((0..1, 'H')));
assert_eq!(scanner.next(), Ok((1..2, 'e')));

assert_eq!(scanner.remaining_text(), "llo");

assert_eq!(scanner.next(), Ok((2..3, 'l')));
assert_eq!(scanner.next(), Ok((3..4, 'l')));
assert_eq!(scanner.next(), Ok((4..5, 'o')));
assert_eq!(scanner.next(), Err((5..5, "")));

assert_eq!(scanner.remaining_text(), "");

pub fn peek(&self) -> ScannerResult<'text, char>

Returns the next char and its Range, if any, without advancing the cursor position.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.peek(), Ok((0..1, 'H')));
assert_eq!(scanner.peek(), Ok((0..1, 'H')));

assert_eq!(scanner.next(), Ok((0..1, 'H')));

assert_eq!(scanner.peek(), Ok((1..2, 'e')));
assert_eq!(scanner.peek(), Ok((1..2, 'e')));

assert_eq!(scanner.remaining_text(), "ello World");

pub fn peek_nth(&self, n: usize) -> ScannerResult<'text, char>

Returns the nth char and its Range, if any, without advancing the cursor position.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.peek_nth(0), Ok((0..1, 'H')));
assert_eq!(scanner.peek_nth(1), Ok((1..2, 'e')));
assert_eq!(scanner.peek_nth(2), Ok((2..3, 'l')));

assert_eq!(scanner.peek_nth(6), Ok((6..7, 'W')));

assert_eq!(scanner.next(), Ok((0..1, 'H')));

assert_eq!(scanner.remaining_text(), "ello World");

pub fn peek_iter(&self) -> CharRangesOffset<'text>

Returns an iterator that produces all the remaining chars and their Ranges, if any, without advancing the cursor position.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.next(), Ok((0..1, 'H')));
assert_eq!(scanner.remaining_text(), "ello World");

let mut peek = scanner.peek_iter();
assert_eq!(peek.next(), Some((1..2, 'e')));
assert_eq!(peek.next(), Some((2..3, 'l')));
assert_eq!(peek.next(), Some((3..4, 'l')));
assert_eq!(scanner.remaining_text(), "ello World");

assert_eq!(scanner.next(), Ok((1..2, 'e')));
assert_eq!(scanner.next(), Ok((2..3, 'l')));
assert_eq!(scanner.remaining_text(), "lo World");

pub fn accept_if<F>(&mut self, f: F) -> ScannerResult<'text, char>where F: FnOnce(char) -> bool,

Advances the scanner cursor and returns the next char and its Range, if f(c) returns true where c is the next character.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.accept_if(char::is_alphabetic), Ok((0..1, 'H')));
assert_eq!(scanner.accept_if(char::is_alphabetic), Ok((1..2, 'e')));
assert_eq!(scanner.accept_if(char::is_alphabetic), Ok((2..3, 'l')));
assert_eq!(scanner.accept_if(char::is_alphabetic), Ok((3..4, 'l')));
assert_eq!(scanner.accept_if(char::is_alphabetic), Ok((4..5, 'o')));
assert_eq!(scanner.accept_if(char::is_alphabetic), Err((5..5, "")));

assert_eq!(scanner.remaining_text(), " World");

pub fn accept_char(&mut self, expected: char) -> ScannerResult<'text, char>

Advances the scanner cursor and returns the next char and its Range, if the next character matches expected.

Example

let mut scanner = Scanner::new("Hello World");

assert_eq!(scanner.accept_char('H'), Ok((0..1, 'H')));
assert_eq!(scanner.accept_char('E'), Err((1..1, "")));
assert_eq!(scanner.accept_char('e'), Ok((1..2, 'e')));
assert_eq!(scanner.accept_char('W'), Err((2..2, "")));

assert_eq!(scanner.remaining_text(), "llo World");

pub fn accept_char_any( &mut self, expected: &[char] ) -> ScannerResult<'text, char>

Advances the scanner cursor and returns the next char and its Range, if the next character matches any char produced by expected.

Panics

Panics in non-optimized builds, if expected is empty.

In optimized builds Err((cursor..cursor, "")) is returned instead.

Example

let mut scanner = Scanner::new("Hello World");

let any = &['H', 'e', 'l', 'o', ' '];
assert_eq!(scanner.accept_char_any(any), Ok((0..1, 'H')));
assert_eq!(scanner.accept_char_any(any), Ok((1..2, 'e')));
assert_eq!(scanner.accept_char_any(any), Ok((2..3, 'l')));
assert_eq!(scanner.accept_char_any(any), Ok((3..4, 'l')));
assert_eq!(scanner.accept_char_any(any), Ok((4..5, 'o')));
assert_eq!(scanner.accept_char_any(any), Ok((5..6, ' ')));
assert_eq!(scanner.accept_char_any(any), Err((6..6, "")));

assert_eq!(scanner.remaining_text(), "World");

pub fn skip_while<F>(&mut self, f: F) -> ScannerItem<&'text str>where F: FnMut(char) -> bool,

Advances the scanner cursor and skips zero-to-many characters, while f(c) returns true, where c is the remaining characters in sequential order.

Returns the string slice and its Range, of the matched (i.e. skipped) characters.

Returns (cursor..cursor, "") if 0 characters were matched (i.e. skipped).

Note: The returned string slice has the same lifetime as the original text, so the scanner can continue to be used while this exists.

Example

let mut scanner = Scanner::new("Hello World");

// Skip all alphabetic characters
assert_eq!(scanner.skip_while(|c| c.is_alphabetic()), (0..5, "Hello"));

// Returns an empty range and an empty string slice
// since 0 characters were skipped
assert_eq!(scanner.skip_while(|c| c.is_alphabetic()), (5..5, ""));

// Skip 1 whitespace character
assert_eq!(scanner.skip_while(char::is_whitespace), (5..6, " "));

assert_eq!(scanner.remaining_text(), "World");

pub fn skip_while_char(&mut self, expected: char) -> ScannerItem<&'text str>

Skips zero-to-many characters matching expected, same as:

scanner.skip_while(|c| c == expected);

pub fn skip_while_char_any( &mut self, expected: &[char] ) -> ScannerItem<&'text str>

Skips zero-to-many characters, which match any character in expected, same as:

scanner.skip_while(|c| expected.contains(&c));

pub fn skip_until<F>(&mut self, f: F) -> ScannerItem<&'text str>where F: FnMut(char) -> bool,

Advances the scanner cursor and skips zero-to-many characters, while f(c) returns false, where c is the remaining characters in sequential order.

Returns the string slice and its Range, of the matched (i.e. skipped) characters.

Returns (cursor..cursor, "") if 0 characters were matched (i.e. skipped).

Note: The returned string slice has the same lifetime as the original text, so the scanner can continue to be used while this exists.

Example

let mut scanner = Scanner::new("Hello World");

// Skip all characters until a whitespace is found
assert_eq!(scanner.skip_until(|c| c.is_whitespace()), (0..5, "Hello"));

// Returns an empty range and an empty string slice
// since 0 characters were skipped
assert_eq!(scanner.skip_until(|c| c.is_whitespace()), (5..5, ""));

// Skip 1 whitespace character
assert_eq!(scanner.skip_until(char::is_alphabetic), (5..6, " "));

assert_eq!(scanner.remaining_text(), "World");

pub fn skip_until_char(&mut self, expected: char) -> ScannerItem<&'text str>

Skips zero-to-many characters, until the next character matches expected, same as:

scanner.skip_until(|c| c == expected);

pub fn skip_until_char_any( &mut self, expected: &[char] ) -> ScannerItem<&'text str>

Skips zero-to-many characters, until the next character match any in expected, same as:

scanner.skip_until(|c| expected.contains(&c));

pub fn skip_whitespace(&mut self) -> ScannerItem<&'text str>

Skips zero-to-many characters, while the next character is a whitespace, same as:

scanner.skip_while(char::is_whitespace);

pub fn scan_with<F>(&mut self, f: F) -> ScannerResult<'text, &'text str>where F: FnOnce(&mut Self) -> ScanResult<'text>,

Advances the cursor if f() returns Ok, otherwise on Err the cursor position is backtracked to before f() was called.

Utility for scanning tokens, where an unexpected character during scanning, should restore the cursor position before the the scan was started.

Additionally, returns Err if f() returns Ok, without advancing the cursor position.

Example

fn scan_word<'text>(scanner: &mut Scanner<'text>) -> Result<(), ScannerItem<&'text str>> {
    // Get next char if alphabetic or return err
    let (first, _c) = scanner.accept_if(char::is_alphabetic)?;
    // Skip zero-to-many alphabetic characters
    let (last, _s) = scanner.skip_while(char::is_alphabetic);
    Ok(())
}

let text = "Hello World";
let mut scanner = Scanner::new(text);

assert_eq!(scanner.scan_with(scan_word), Ok((0..5, "Hello")));
assert_eq!(scanner.scan_with(scan_word), Err((5..5, "")));
assert_eq!(scanner.next(), Ok((5..6, ' ')));
assert_eq!(scanner.scan_with(scan_word), Ok((6..11, "World")));

Trait Implementations

impl<'text> Clone for Scanner<'text>

fn clone(&self) -> Scanner<'text>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'text> Debug for Scanner<'text>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'text> RustScannerExt<'text> for Scanner<'text>

fn scan_rust_line_comment(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust line comment.

fn scan_rust_block_comment(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust block comment.

fn scan_rust_identifier(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust identifier.

fn scan_rust_raw_identifier(&mut self) -> ScannerResult<'text, &'text str>

Scans a single raw Rust identifier.

fn scan_rust_char(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust character.

fn scan_rust_string(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust string.

fn scan_rust_raw_string(&mut self) -> ScannerResult<'text, &'text str>

Scans a single raw Rust string.

fn scan_rust_int_dec(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust integer decimal literal.

fn scan_rust_int_hex(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust integer hex literal.

fn scan_rust_int_oct(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust integer octal literal.

fn scan_rust_int_bin(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust integer binary literal.

fn scan_rust_float(&mut self) -> ScannerResult<'text, &'text str>

Scans a single Rust floating-point literal.