libbf/token/
mod.rs

1//! Token related definitions.
2use crate::error::ParseError;
3
4pub mod simple;
5
6/// A token type.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum TokenType {
9    /// pointer increment (Brainfuck: '>')
10    PInc,
11    /// pointer decrement (Brainfuck: '<)
12    PDec,
13    /// data increment (Brainfuck: '+')
14    DInc,
15    /// data decrement (Brainfuck: '-')
16    DDec,
17    /// output (Brainfuck: '.')
18    Output,
19    /// input (Brainfuck: ',')
20    Input,
21    /// loop head (Brainfuck: '[')
22    LoopHead,
23    /// loop tail (Brainfuck: ']')
24    LoopTail,
25}
26
27/// A token information.
28#[derive(Debug, PartialEq, Eq)]
29pub struct TokenInfo {
30    /// The token type. `None` means the EOF.
31    pub token_type: Option<TokenType>,
32    /// The position of the token in the source string which is counted in Unicode scalar units.
33    /// If `token_type` is `None`, this field points to the position of the EOF.
34    pub pos_in_chars: usize,
35}
36
37/// A tokenizer trait.
38///
39/// This trait generates a [`TokenStream`] from a source string.
40pub trait Tokenizer<'a> {
41    type Stream: TokenStream;
42
43    fn token_stream(&'a self, source: &'a str) -> Self::Stream;
44}
45
46/// A token stream trait.
47///
48/// This trait iterates over tokens in the source string.
49///
50/// # Note
51///
52/// This is not related with the [`Iterator`] trait.
53pub trait TokenStream {
54    fn next(&mut self) -> Result<TokenInfo, ParseError>;
55}
libbf/token/mod.rs

libbf/token/
mod.rs
