do_not_use_antlr_rust/
token_source.rs

1use crate::int_stream::IntStream;
2use crate::token_factory::TokenFactory;
3
4/// Produces tokens to be used by parser.
5/// `TokenStream` implementations are responsible for buffering tokens for parser lookahead
6pub trait TokenSource<'input> {
7    /// TokenFactory this token source produce tokens with
8    type TF: TokenFactory<'input> + 'input;
9    /// Return a {@link Token} object from your input stream (usually a
10    /// {@link CharStream}). Do not fail/return upon lexing error; keep chewing
11    /// on the characters until you get a good one; errors are not passed through
12    /// to the parser.
13    fn next_token(&mut self) -> <Self::TF as TokenFactory<'input>>::Tok;
14    /**
15     * Get the line number for the current position in the input stream. The
16     * first line in the input is line 1.
17     *
18     * Returns the line number for the current position in the input stream, or
19     * 0 if the current token source does not track line numbers.
20     */
21    fn get_line(&self) -> isize { 0 }
22    /**
23     * Get the index into the current line for the current position in the input
24     * stream. The first character on a line has position 0.
25     *
26     * Returns the line number for the current position in the input stream, or
27     * -1 if the current token source does not track character positions.
28     */
29    fn get_char_position_in_line(&self) -> isize { -1 }
30
31    /// Returns underlying input stream
32    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream>;
33
34    /// Returns string identifier of underlying input e.g. file name
35    fn get_source_name(&self) -> String;
36    //    fn set_token_factory<'c: 'b>(&mut self, f: &'c TokenFactory);
37    /// Gets the `TokenFactory` this token source is currently using for
38    /// creating `Token` objects from the input.
39    ///
40    /// Required by `Parser` for creating missing tokens.
41    fn get_token_factory(&self) -> &'input Self::TF;
42}
43
44// allows user to call parser with &mut reference to Lexer
45impl<'input, T> TokenSource<'input> for &mut T
46where
47    T: TokenSource<'input>,
48{
49    type TF = T::TF;
50    #[inline(always)]
51    fn next_token(&mut self) -> <Self::TF as TokenFactory<'input>>::Tok { (**self).next_token() }
52
53    #[inline(always)]
54    fn get_line(&self) -> isize { (**self).get_line() }
55
56    #[inline(always)]
57    fn get_char_position_in_line(&self) -> isize { (**self).get_char_position_in_line() }
58
59    #[inline(always)]
60    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream> { (**self).get_input_stream() }
61
62    #[inline(always)]
63    fn get_source_name(&self) -> String { (**self).get_source_name() }
64
65    #[inline(always)]
66    fn get_token_factory(&self) -> &'input Self::TF { (**self).get_token_factory() }
67}
68
69// / adaptor to feed parser with existing tokens
70// pub struct IterTokenSource<S, F> where S: Iterator, S::Item: Token, F: TokenFactory<Tok=S::Item> {
71//     iter: S,
72//     fact: F,
73// }
74//
75// impl<S, F> TokenSource for IterTokenSource<S, F> where S: Iterator, S::Item: Token, F: TokenFactory<Tok=S::Item> {
76//     type Tok = S::Item;
77//
78//     fn next_token(&mut self) -> Box<Self::Tok> {
79//         self.iter.next().map(Box::new).unwrap_or_else(
80//             || self.get_token_factory().create(
81//                 None,
82//                 EOF,
83//                 TOKEN_DEFAULT_CHANNEL,
84//                 -1,
85//                 -1,
86//                 self.get_line(),
87//                 self.get_char_position_in_line(),
88//             )
89//         )
90//     }
91//
92//     fn get_line(&self) -> isize {
93//         0
94//     }
95//
96//     fn get_char_position_in_line(&self) -> isize {
97//         -1
98//     }
99//
100//     fn get_input_stream(&mut self) -> Option<&mut dyn CharStream> {
101//         None
102//     }
103//
104//     fn get_source_name(&self) -> String {
105//         "<iterator>".to_string()
106//     }
107//
108//     fn get_token_factory(&self) -> &dyn TokenFactory<Tok=Self::Tok> {
109//         &self.fact
110//     }
111// }
do_not_use_antlr_rust/token_source.rs

do_not_use_antlr_rust/
token_source.rs
