castle_tokenizer/
tokenizable.rs

1
2
3
4use castle_types::CastleError;
5
6use crate::{Token, TokenKind};
7pub trait Tokenizable {
8    fn next(&mut self, skip_line_terminators: bool) -> Result<Option<Token>, CastleError>;
9    fn peek_n(
10        &mut self,
11        skip_n: usize,
12        skip_line_terminators: bool,
13    ) -> Result<Option<&Token>, CastleError>;
14
15    fn peek(&mut self, skip_line_terminators: bool) -> Result<Option<&Token>, CastleError> {
16        self.peek_n(0, skip_line_terminators)
17    }
18    fn peek_expect(&mut self, skip_line_terminators: bool) -> Result<&Token, CastleError> {
19        match self.peek(skip_line_terminators)? {
20            Some(token) => Ok(token),
21            None => Err(CastleError::AbruptEOF("Expected token but got EOF".into())),
22        }
23    }
24
25    fn peek_token_kind(&mut self, skip_line_terminators: bool) -> Result<Option<&TokenKind>, CastleError> {
26        match self.peek(skip_line_terminators)? {
27            Some(token) => Ok(Some(&token.kind)),
28            None => Ok(None),
29        }
30    }
31}
castle_tokenizer/tokenizable.rs

castle_tokenizer/
tokenizable.rs
