Struct TokenReader

pub struct TokenReader {
    pub cursor: usize,
    /* private fields */
}

A simple character reader that is useful for creating lexers

Fields

cursor: usize

A cursor. Just a byte index, useful for spanning and then codespan_reporting

Implementations

impl TokenReader

pub fn new(source: Rc<str>) -> Self

Create new TokenReader. Just requires a source as Rc, can be provided by the crate::Codebase

Examples found in repository

examples/simple_c_lexer.rs (line 15)

pub fn main() {
    let args = std::env::args().collect::<Vec<_>>();
    if args.len() != 2 {
        eprintln!("Usage: cargo run --example simple_c_lexer -- sample.c");
        return;
    }

    let code = std::fs::read_to_string(&args[1]).unwrap();
    let mut codebase = Codebase::new();
    let key = codebase.add(args[1].clone(), code);

    let mut reader = TokenReader::new(codebase.get(key).unwrap().source().clone());
    while let Some(char) = reader.next_char() {
        if chars::is_digit(char) {
            // * Number
            println!("Number {}", reader.next_token(chars::is_digit, char));
        } else if chars::is_ident_start(char) {
            // * Identifier
            println!(
                "Ident {}",
                reader.next_token(chars::is_ident_continue, char)
            );
        } else if char == '"' {
            // * String
            let mut string = String::new();
            loop {
                match reader.next_char() {
                    // Escape sequences
                    Some('\\') => match reader.next_char() {
                        Some('n') => string.push('\n'),
                        Some(char) => string.push(char),
                        None => codebase.emit( // Technically won't happen
                            Diagnostic::error()
                                .with_message("expected escape code")
                                .with_labels(vec![Label::primary(
                                    key,
                                    reader.cursor..reader.cursor + 1,
                                )
                                .with_message("here")]),
                        ),
                    },

                    // Quote
                    Some('"') => break,

                    // Char
                    Some(char) if char != '\n' => string.push(char),
                    _ => {
                        codebase.emit(
                            Diagnostic::error()
                                .with_message("unterminated string")
                                .with_labels(vec![Label::primary(
                                    key,
                                    reader.cursor..reader.cursor + 1,
                                )
                                .with_message("here")]),
                        );
                        break;
                    }
                }
            }
            println!("String {:?}", string);
        } else if ['(', ')', '{', '}', ';'].contains(&char) {
            // * Operator
            println!("Operator {}", char);
        }
    }
}

pub fn peek_char(&mut self) -> Option<char>

Peek a char

pub fn next_char(&mut self) -> Option<char>

Peek a char, return it and move cursor forward

Examples found in repository

examples/simple_c_lexer.rs (line 16)

pub fn main() {
    let args = std::env::args().collect::<Vec<_>>();
    if args.len() != 2 {
        eprintln!("Usage: cargo run --example simple_c_lexer -- sample.c");
        return;
    }

    let code = std::fs::read_to_string(&args[1]).unwrap();
    let mut codebase = Codebase::new();
    let key = codebase.add(args[1].clone(), code);

    let mut reader = TokenReader::new(codebase.get(key).unwrap().source().clone());
    while let Some(char) = reader.next_char() {
        if chars::is_digit(char) {
            // * Number
            println!("Number {}", reader.next_token(chars::is_digit, char));
        } else if chars::is_ident_start(char) {
            // * Identifier
            println!(
                "Ident {}",
                reader.next_token(chars::is_ident_continue, char)
            );
        } else if char == '"' {
            // * String
            let mut string = String::new();
            loop {
                match reader.next_char() {
                    // Escape sequences
                    Some('\\') => match reader.next_char() {
                        Some('n') => string.push('\n'),
                        Some(char) => string.push(char),
                        None => codebase.emit( // Technically won't happen
                            Diagnostic::error()
                                .with_message("expected escape code")
                                .with_labels(vec![Label::primary(
                                    key,
                                    reader.cursor..reader.cursor + 1,
                                )
                                .with_message("here")]),
                        ),
                    },

                    // Quote
                    Some('"') => break,

                    // Char
                    Some(char) if char != '\n' => string.push(char),
                    _ => {
                        codebase.emit(
                            Diagnostic::error()
                                .with_message("unterminated string")
                                .with_labels(vec![Label::primary(
                                    key,
                                    reader.cursor..reader.cursor + 1,
                                )
                                .with_message("here")]),
                        );
                        break;
                    }
                }
            }
            println!("String {:?}", string);
        } else if ['(', ')', '{', '}', ';'].contains(&char) {
            // * Operator
            println!("Operator {}", char);
        }
    }
}

pub fn next_char_if(&mut self, pred: impl FnOnce(char) -> bool) -> Option<char>

Self::next_char() but advances if predicate

pub fn next_token( &mut self, pred: impl Fn(char) -> bool, prefix: impl Into<String>, ) -> String

Takes characters and adds then to the buffer while predicate

Examples found in repository

examples/simple_c_lexer.rs (line 19)

pub fn main() {
    let args = std::env::args().collect::<Vec<_>>();
    if args.len() != 2 {
        eprintln!("Usage: cargo run --example simple_c_lexer -- sample.c");
        return;
    }

    let code = std::fs::read_to_string(&args[1]).unwrap();
    let mut codebase = Codebase::new();
    let key = codebase.add(args[1].clone(), code);

    let mut reader = TokenReader::new(codebase.get(key).unwrap().source().clone());
    while let Some(char) = reader.next_char() {
        if chars::is_digit(char) {
            // * Number
            println!("Number {}", reader.next_token(chars::is_digit, char));
        } else if chars::is_ident_start(char) {
            // * Identifier
            println!(
                "Ident {}",
                reader.next_token(chars::is_ident_continue, char)
            );
        } else if char == '"' {
            // * String
            let mut string = String::new();
            loop {
                match reader.next_char() {
                    // Escape sequences
                    Some('\\') => match reader.next_char() {
                        Some('n') => string.push('\n'),
                        Some(char) => string.push(char),
                        None => codebase.emit( // Technically won't happen
                            Diagnostic::error()
                                .with_message("expected escape code")
                                .with_labels(vec![Label::primary(
                                    key,
                                    reader.cursor..reader.cursor + 1,
                                )
                                .with_message("here")]),
                        ),
                    },

                    // Quote
                    Some('"') => break,

                    // Char
                    Some(char) if char != '\n' => string.push(char),
                    _ => {
                        codebase.emit(
                            Diagnostic::error()
                                .with_message("unterminated string")
                                .with_labels(vec![Label::primary(
                                    key,
                                    reader.cursor..reader.cursor + 1,
                                )
                                .with_message("here")]),
                        );
                        break;
                    }
                }
            }
            println!("String {:?}", string);
        } else if ['(', ')', '{', '}', ';'].contains(&char) {
            // * Operator
            println!("Operator {}", char);
        }
    }
}

Trait Implementations

impl Debug for TokenReader

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.