dbt_antlr4/
token_source.rs

1use crate::int_stream::IntStream;
2use crate::token_factory::TokenFactory;
3
4/// Produces tokens to be used by parser (aka Lexer).
5///
6/// `TokenStream` implementations are responsible for buffering tokens for
7/// parser lookahead
8pub trait TokenSource<'input, 'arena, TF>
9where
10    'input: 'arena,
11    TF: TokenFactory<'input, 'arena> + 'arena,
12{
13    /// Return a {@link Token} object from your input stream (usually a {@link
14    /// CharStream}). Do not fail/return upon lexing error; keep chewing on the
15    /// characters until you get a good one; errors are not passed through to
16    /// the parser.
17    fn next_token(&mut self) -> &'arena mut TF::Tok;
18
19    /// Get the line number for the current position in the input stream. The
20    /// first line in the input is line 1.
21    ///
22    /// Returns the line number for the current position in the input stream,
23    /// or 0 if the current token source does not track line numbers.
24    fn get_line(&self) -> u32 {
25        0
26    }
27
28    /// Get the index into the current line for the current position in the
29    /// input stream. The first character on a line has position 0.
30    ///
31    /// Returns the line number for the current position in the input stream, or
32    /// -1 if the current token source does not track character positions.
33    fn get_char_position_in_line(&self) -> i32 {
34        -1
35    }
36
37    /// Returns underlying input stream
38    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream>;
39
40    /// Returns string identifier of underlying input e.g. file name
41    fn get_source_name(&self) -> String;
42
43    /// Gets the `TokenFactory` this token source is currently using for
44    /// creating `Token` objects from the input.
45    ///
46    /// Required by `Parser` for creating missing tokens.
47    fn get_token_factory(&self) -> &TF;
48
49    fn get_dfa_string(&self) -> String;
50}
51
52// allows user to call parser with &mut reference to Lexer
53impl<'input, 'arena, T, TF> TokenSource<'input, 'arena, TF> for &mut T
54where
55    'input: 'arena,
56    T: TokenSource<'input, 'arena, TF>,
57    TF: TokenFactory<'input, 'arena> + 'arena,
58{
59    #[inline(always)]
60    fn next_token(&mut self) -> &'arena mut TF::Tok {
61        (**self).next_token()
62    }
63
64    #[inline(always)]
65    fn get_line(&self) -> u32 {
66        (**self).get_line()
67    }
68
69    #[inline(always)]
70    fn get_char_position_in_line(&self) -> i32 {
71        (**self).get_char_position_in_line()
72    }
73
74    #[inline(always)]
75    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream> {
76        (**self).get_input_stream()
77    }
78
79    #[inline(always)]
80    fn get_source_name(&self) -> String {
81        (**self).get_source_name()
82    }
83
84    #[inline(always)]
85    fn get_token_factory(&self) -> &TF {
86        (**self).get_token_factory()
87    }
88
89    #[inline(always)]
90    fn get_dfa_string(&self) -> String {
91        (**self).get_dfa_string()
92    }
93}
94
95// / adaptor to feed parser with existing tokens
96// pub struct IterTokenSource<S, F> where S: Iterator, S::Item: Token, F: TokenFactory<Tok=S::Item> {
97//     iter: S,
98//     fact: F,
99// }
100//
101// impl<S, F> TokenSource for IterTokenSource<S, F> where S: Iterator, S::Item: Token, F: TokenFactory<Tok=S::Item> {
102//     type Tok = S::Item;
103//
104//     fn next_token(&mut self) -> Box<Self::Tok> {
105//         self.iter.next().map(Box::new).unwrap_or_else(
106//             || self.get_token_factory().create(
107//                 None,
108//                 EOF,
109//                 TOKEN_DEFAULT_CHANNEL,
110//                 -1,
111//                 -1,
112//                 self.get_line(),
113//                 self.get_char_position_in_line(),
114//             )
115//         )
116//     }
117//
118//     fn get_line(&self) -> isize {
119//         0
120//     }
121//
122//     fn get_char_position_in_line(&self) -> isize {
123//         -1
124//     }
125//
126//     fn get_input_stream(&mut self) -> Option<&mut dyn CharStream> {
127//         None
128//     }
129//
130//     fn get_source_name(&self) -> String {
131//         "<iterator>".to_string()
132//     }
133//
134//     fn get_token_factory(&self) -> &dyn TokenFactory<Tok=Self::Tok> {
135//         &self.fact
136//     }
137// }
dbt_antlr4/token_source.rs

dbt_antlr4/
token_source.rs
