alkale/common/
mod.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
//! This module contains many helpful "built-in" helper methods added to [TokenizerContext].

use std::ops::ControlFlow;

use crate::{span::Span, token::Token, TokenizerContext};

pub mod error;
pub mod identifier;
pub mod numeric;
pub mod string;
pub mod structure;

impl<S: Iterator<Item = char>, T> TokenizerContext<S, T> {
    /// This function will repeatedly skip characters until EOF or until it finds a character
    /// that matches the input predicate.
    ///
    /// While this can be used for any purpose, this is intended to be used for error recovery
    /// to bring the tokenizer into a "safe" position to resume tokenization.
    ///
    /// Also see: [skip_until]
    pub fn recover_with(&mut self, predicate: impl Fn(char) -> bool) {
        while let Some(char) = self.peek() {
            if predicate(char) {
                break;
            } else {
                self.skip();
            }
        }
    }

    /// This function will repeatedly skip characters until EOF or until it finds a character
    /// that matches in input character.
    pub fn skip_until(&mut self, match_character: char) {
        while let Some(char) = self.peek() {
            if char == match_character {
                break;
            } else {
                self.skip();
            }
        }
    }

    /// Takes an input predicate and executes it immediately. The result of the predicate will be
    /// returned alongside a [Span] that captures the entire affected region of source code of the predicate.
    ///
    /// In other words, this method creates a span between the source code position before and after the
    /// predicate is executed. The span is then returned alongside the predicate's result.
    ///
    /// The predicate will be supplied with the mutable [TokenizerContext] reference that was supplied to prevent
    /// a situation where two mutable references are needed.
    pub fn capture_span<R>(
        &mut self,
        predicate: impl FnOnce(&mut TokenizerContext<S, T>) -> R,
    ) -> (R, Span) {
        let start_position = self.span().clone();

        let out = predicate(self);

        let span = start_position.up_to(self.span());

        (out, span)
    }

    /// If the next character matches the input predicate, append
    /// it to the String argument. Repeat until EOF or until next
    /// character doesn't match.
    ///
    /// Returns a [Span] over all of the consumed characters, or [None]
    /// if no characters were consumed.
    pub fn read_into_while(
        &mut self,
        string: &mut String,
        predicate: impl Fn(&char) -> bool,
    ) -> Option<Span> {
        let (_, span) = self.capture_span(|context| {
            while let Some(next_char) = context.peek() {
                if predicate(&next_char) {
                    context.skip();
                    string.push(next_char);
                } else {
                    break;
                }
            }
        });

        if !span.is_empty() {
            Some(span)
        } else {
            None
        }
    }

    /// Continuously peeks the next character and executes the predicate function.
    /// If the predicate function returns [ControlFlow::Break], stop and return the accumulator.
    /// If the predicate function returns [ControlFlow::Continue], skip the character that was peeked and repeat.
    ///
    /// The predicate function is supplied the peeked character and a mutable reference to the current accumulator.
    /// The predicate will not be called if EOF is reached.
    pub fn fold<A>(
        &mut self,
        mut accumulator: A,
        predicate: impl Fn(char, &mut A) -> ControlFlow<(), ()>,
    ) -> A {
        while let Some(char) = self.peek() {
            if predicate(char, &mut accumulator).is_break() {
                break;
            }

            self.skip();
        }

        accumulator
    }

    /// Peeks a single character and passes it into the predicate. If the predicate returns
    /// [None], nothing happens. If the predicate returns [Some] token data, the character
    /// will be consumed and its span used to construct a [Token] alongside the returned data—
    /// the constructed token will then be pushed to this context's token list.
    ///
    /// This method will return `true` if a token was pushed, and `false` if nothing occured.
    /// If this context has no characters left to iterate, this method will do nothing.
    ///
    /// This method is intended to be used to easily handle single-character tokens.
    pub fn map_single_char_token(&mut self, predicate: impl FnOnce(char) -> Option<T>) -> bool {
        // If there are more characters...
        if let Some(ch) = self.peek() {
            // If the next character matches the predicate...
            if let Some(data) = predicate(ch) {
                self.push_token(Token::new(data, self.span().clone()));
                self.skip();

                return true;
            }
        }

        false
    }
}