1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111

use crate::int_stream::IntStream;
use crate::token_factory::TokenFactory;

/// Produces tokens to be used by parser.
/// `TokenStream` implementations are responsible for buffering tokens for parser lookahead
pub trait TokenSource<'input> {
    /// TokenFactory this token source produce tokens with
    type TF: TokenFactory<'input> + 'input;
    /// Return a {@link Token} object from your input stream (usually a
    /// {@link CharStream}). Do not fail/return upon lexing error; keep chewing
    /// on the characters until you get a good one; errors are not passed through
    /// to the parser.
    fn next_token(&mut self) -> <Self::TF as TokenFactory<'input>>::Tok;
    /**
     * Get the line number for the current position in the input stream. The
     * first line in the input is line 1.
     *
     * Returns the line number for the current position in the input stream, or
     * 0 if the current token source does not track line numbers.
     */
    fn get_line(&self) -> isize { 0 }
    /**
     * Get the index into the current line for the current position in the input
     * stream. The first character on a line has position 0.
     *
     * Returns the line number for the current position in the input stream, or
     * -1 if the current token source does not track character positions.
     */
    fn get_char_position_in_line(&self) -> isize { -1 }

    /// Returns underlying input stream
    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream>;

    /// Returns string identifier of underlying input e.g. file name
    fn get_source_name(&self) -> String;
    //    fn set_token_factory<'c: 'b>(&mut self, f: &'c TokenFactory);
    /// Gets the `TokenFactory` this token source is currently using for
    /// creating `Token` objects from the input.
    ///
    /// Required by `Parser` for creating missing tokens.
    fn get_token_factory(&self) -> &'input Self::TF;
}

// allows user to call parser with &mut reference to Lexer
impl<'input, T> TokenSource<'input> for &mut T
where
    T: TokenSource<'input>,
{
    type TF = T::TF;
    #[inline(always)]
    fn next_token(&mut self) -> <Self::TF as TokenFactory<'input>>::Tok { (**self).next_token() }

    #[inline(always)]
    fn get_line(&self) -> isize { (**self).get_line() }

    #[inline(always)]
    fn get_char_position_in_line(&self) -> isize { (**self).get_char_position_in_line() }

    #[inline(always)]
    fn get_input_stream(&mut self) -> Option<&mut dyn IntStream> { (**self).get_input_stream() }

    #[inline(always)]
    fn get_source_name(&self) -> String { (**self).get_source_name() }

    #[inline(always)]
    fn get_token_factory(&self) -> &'input Self::TF { (**self).get_token_factory() }
}

// / adaptor to feed parser with existing tokens
// pub struct IterTokenSource<S, F> where S: Iterator, S::Item: Token, F: TokenFactory<Tok=S::Item> {
//     iter: S,
//     fact: F,
// }
//
// impl<S, F> TokenSource for IterTokenSource<S, F> where S: Iterator, S::Item: Token, F: TokenFactory<Tok=S::Item> {
//     type Tok = S::Item;
//
//     fn next_token(&mut self) -> Box<Self::Tok> {
//         self.iter.next().map(Box::new).unwrap_or_else(
//             || self.get_token_factory().create(
//                 None,
//                 EOF,
//                 TOKEN_DEFAULT_CHANNEL,
//                 -1,
//                 -1,
//                 self.get_line(),
//                 self.get_char_position_in_line(),
//             )
//         )
//     }
//
//     fn get_line(&self) -> isize {
//         0
//     }
//
//     fn get_char_position_in_line(&self) -> isize {
//         -1
//     }
//
//     fn get_input_stream(&mut self) -> Option<&mut dyn CharStream> {
//         None
//     }
//
//     fn get_source_name(&self) -> String {
//         "<iterator>".to_string()
//     }
//
//     fn get_token_factory(&self) -> &dyn TokenFactory<Tok=Self::Tok> {
//         &self.fact
//     }
// }