cpp_rs 0.1.0

// Reference:
// [c11]: http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf

use log::trace;
use std::borrow::Cow;

mod char_source;
mod peek_adapter;

use char_source::CharSource;
pub use char_source::{StrSource, StringSource};
use peek_adapter::PeekAdapter;

#[cfg(test)]
mod tests;

pub type OwningTokenizer = Tokenizer<'static, StringSource>;
pub type StrTokenizer<'s> = Tokenizer<'s, StrSource<'s>>;

/// A variant of the `try!` macro which is designed to handle the various
/// state machine actions. This works just like `try!` but propagates both
/// Err and Token variants.
macro_rules! try_action {
    ($e:expr) => {
        match $e {
            ActionResult::Ok(good) => good,
            ActionResult::Err(e) => return StateMachineAction::Emit(Err(e)),
            ActionResult::Token(t) => return StateMachineAction::Emit(Ok(t)),
        }
    };
}

#[derive(Debug)]
pub struct Tokenizer<'s, S: CharSource<'s>> {
    state: State,

    is_header_valid: bool,
    hit_eof: bool,

    /// The char stream handles all the Multi-peeking grabbing of indices
    char_input: PeekAdapter<S>,

    _lifetime: std::marker::PhantomData<&'s ()>,
}

#[warn(missing_docs)]
impl<'s, S: CharSource<'s>> Tokenizer<'s, S> {
    /// Create a new tokenizer based on the input string. This maintans all
    /// the state needed to tokenize a string into Preprocessor tokens
    pub fn new<I: Into<S>>(input: I) -> Tokenizer<'s, S> {
        Tokenizer {
            state: State::Start,

            is_header_valid: false,
            hit_eof: false,

            char_input: PeekAdapter::new(input.into()),

            _lifetime: std::marker::PhantomData,
        }
    }

    /// Gets the next token from the input string. If there is an error in the
    /// token string it will return that error. An error does not indicate that
    /// the token string has terminated. If another token is requested and it
    /// can recover it will return a token. Otherwise it will return an error
    /// indicating it cannot recover.
    ///
    /// ```
    /// use cpp_rs::lexer::{Tokenizer, Token, TokenType, StrSource};
    /// use std::borrow::Cow;
    ///
    /// let mut tokenizer : Tokenizer<StrSource<'static>> = Tokenizer::new("<");
    ///
    /// assert_eq!(
    ///     tokenizer.get_next_token().unwrap(),
    ///     Token{
    ///         token_type: TokenType::Punctuator,
    ///         start_byte:0,
    ///         end_byte:0,
    ///         text: Cow::Owned("<".into()),
    ///     }
    /// );
    /// ```
    ///
    /// Once an EOF token has been generated EOF will continue to be generated
    /// forever.
    pub fn get_next_token(&mut self) -> Result<Token<'s>, Error> {
        loop {
            trace!("Entering state: {:?}", self.state);
            let temp_state = self.state.clone();

            let action = match temp_state {
                State::Start => self.handle_start(),
                State::QuotedHeaderName => self.handle_quoted_header_name(),
                State::BracketedHeaderName => self.handle_bracketed_header_name(),
                State::IdentOrString => self.handle_ident_or_string(),
                State::IdentOrStringU8 => self.handle_ident_or_string_u8(),
                State::Identifier => self.handle_identifier(),
                State::StringLiteral => self.handle_string_literal(),
                State::UCN {
                    to_count,
                    return_to,
                } => self.handle_ucn(to_count, return_to),
                State::PPNumber => self.handle_ppnumber(),
                State::PPNumberExponent => self.handle_ppnumber_exponent(),
                State::PPNumberOrDot => self.handle_ppnumber_or_dot(),

                State::PunctGTStart => self.handle_punct_gt(),
                State::PunctLTStart => self.handle_punct_lt(),
                State::PunctAndEqual => self.handle_punct_and_equal(),
                State::PunctPercent => self.handle_punct_percent(),
                State::PunctColon => self.handle_punct_colon(),
                State::PunctMinus => self.handle_punct_minus(),
                State::PunctDoubleOrEq(c) => self.handle_punct_double_or_equal(c),
                State::PunctDouble(c) => self.handle_punct_double(c),
                State::AltPound => self.handle_alt_pound(),
                State::Error(other) => self.handle_error(*other),
                State::Whitespace => self.handle_whitespace(),
                State::EOL => self.handle_eol(),
                State::DivideOrComment => self.handle_divide_or_comment(),
                State::LineComment => self.handle_line_comment(),
                State::BlockComment => self.handle_block_comment(),
            };

            self.reset_peek();
            match action {
                StateMachineAction::Continue => {}
                StateMachineAction::Emit(e) => {
                    trace!("Emitting: {:?}", e);
                    return e;
                }
            }
        }
    }

    fn handle_start(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::EOF));

        match next_char {
            // Line endings are `\n` and `\r`
            '\n' | '\r' => {
                self.transition(State::EOL);
            }

            // Whitespace is anything which isn't a newline
            _ if next_char.is_ascii_whitespace() => {
                try_action!(self.consume(1));
                self.transition(State::Whitespace);
            }

            // [c11]: §6.4 ¶4
            // If we see a quote and are in a valid state for a HeaderName token we transition to the
            // quoted header name state.
            '"' if self.is_header_valid => {
                try_action!(self.consume(1));
                self.transition(State::QuotedHeaderName);
            }

            // [c11]: §6.4 ¶4
            // If we see a bracket and are in a valid state for a HeaderName token we transition to the
            // bracketed header name state.
            '<' if self.is_header_valid => {
                try_action!(self.consume(1));
                self.transition(State::BracketedHeaderName);
            }

            // [c11]: §6.4.5 ¶1
            // If we see any of the following symbols it is possible
            // we are starting a string literal or an identifier.
            'u' | 'U' | 'L' => {
                try_action!(self.consume(1));
                self.transition(State::IdentOrString);
            }

            // [c11]: §6.4.5 ¶1
            // If we start with a quote we are as string literal
            '"' => {
                try_action!(self.consume(1));
                self.transition(State::StringLiteral);
            }

            // [c11]: §6.4.2.1 ¶1
            // An underline can start an identifier
            '_' => {
                try_action!(self.consume(1));
                self.transition(State::Identifier);
            }

            // [c11]: §6.4.2.1 ¶1
            // Any alphabetic character can start an identifier
            _ if next_char.is_ascii_alphabetic() => {
                try_action!(self.consume(1));
                self.transition(State::Identifier);
            }

            // [c11]: §6.4.8 ¶1
            // Anything which starts with a digit is going to be a
            // Preprocessor Number
            _ if next_char.is_ascii_digit() => {
                try_action!(self.consume(1));
                self.transition(State::PPNumber);
            }

            // [c11]: §6.4.8 ¶1
            // This can either be a PPNumber or can be a punctuator
            // based on whichever is going to be longer so we have
            // to check.
            '.' => {
                try_action!(self.consume(1));
                self.transition(State::PPNumberOrDot);
            }

            // [c11]: §6.4.6 ¶1
            // Simple punctuators don't have any follow up characters
            '[' | ']' | '(' | ')' | '{' | '}' | ';' | ',' | '?' => {
                try_action!(self.consume(1));
                return self.emit(TokenType::Punctuator);
            }

            // [c11]: §6.4.6 ¶1
            // Punctuators which start with '<' this includes
            //  * <
            //  * <=
            //  * <<
            //  * <<=
            //  * <:
            //  * <%
            '<' => {
                try_action!(self.consume(1));
                self.transition(State::PunctLTStart);
            }

            // [c11]: §6.4.6 ¶1
            // Punctuators which start with '>' this includes
            //  * >
            //  * >=
            //  * >>
            //  * >>=
            '>' => {
                try_action!(self.consume(1));
                self.transition(State::PunctGTStart);
            }

            // [c11]: §6.4.6 ¶1
            // Puncuators which start with '%' including:
            //  * %
            //  * %=
            //  * %>
            //  * %:
            //  * %:%:
            '%' => {
                try_action!(self.consume(1));
                self.transition(State::PunctPercent);
            }

            // [c11]: §6.4.6 ¶1
            // Punctuators which start with ':' include:
            //  * :
            //  * :%
            //  * :%:%
            //  * :>
            ':' => {
                try_action!(self.consume(1));
                self.transition(State::PunctColon);
            }

            // [c11]: §6.4.6 ¶1
            // Punctuators which start with '-' include:
            //  * -
            //  * --
            //  * -=
            //  * ->
            '-' => {
                try_action!(self.consume(1));
                self.transition(State::PunctMinus);
            }

            // [c11]: §6.4.6 ¶1
            // Punctuators which only exist on their own as a pair or as part
            // of an assignment operator
            x @ '+' | x @ '|' | x @ '&' => {
                try_action!(self.consume(1));
                self.transition(State::PunctDoubleOrEq(x));
            }

            // [c11]: §6.4.6 ¶1
            // Handle all the punctuators which can be themselves or with an
            // equals only:
            //  * ^ | ^=
            //  * * | *=
            //  * ! | !=
            //  * = | ==
            '^' | '*' | '!' | '=' | '~' => {
                try_action!(self.consume(1));
                self.transition(State::PunctAndEqual);
            }

            // [c11]: §6.4.6 ¶1
            // Handles things which start with a slash
            //  * / | /=
            //  * //
            //  * /*
            '/' => {
                try_action!(self.consume(1));
                self.transition(State::DivideOrComment)
            }

            c @ '#' => {
                try_action!(self.consume(1));
                self.transition(State::PunctDouble(c));
            }
            _ => unimplemented!(),
        }
        StateMachineAction::Continue
    }

    // When we enter this state we will have consumed one '<' in a
    // header name valid situation.
    //
    // Consumes: [^"\n]*>
    //
    // On Eof: Throws an early EOF error
    //
    fn handle_bracketed_header_name(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek());

        match next_char {
            '\n' => {
                self.transition(State::Error(Box::new(State::QuotedHeaderName)));
                StateMachineAction::Emit(Err(Error::UnexpectedCharacter {
                    found: '\n',
                    expected: vec!['>'],
                }))
            }

            '>' => {
                self.transition(State::Start);
                try_action!(self.consume(1));
                self.emit(TokenType::HeaderName)
            }

            _ => {
                try_action!(self.consume(1));
                StateMachineAction::Continue
            }
        }
    }

    // When we enter this state we will have consumed one '"' in a
    // header name valid situation.
    //
    // Consumes: [^"\n]*"
    //
    // On Eof: Throws an early EOF error
    //
    fn handle_quoted_header_name(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek());

        match next_char {
            '\n' => {
                self.transition(State::Error(Box::new(State::QuotedHeaderName)));
                StateMachineAction::Emit(Err(Error::UnexpectedCharacter {
                    found: '\n',
                    expected: vec!['"'],
                }))
            }

            '"' => {
                self.transition(State::Start);
                try_action!(self.consume(1));
                self.emit(TokenType::HeaderName)
            }

            _ => {
                try_action!(self.consume(1));
                StateMachineAction::Continue
            }
        }
    }

    // When we enter this state we will have consumed one of [u,U,L]
    // which can either start an identifier or can be a string
    // literal encoding-prefix
    //
    // Consumes: ("|8)?
    //
    // On EOF: Emit an Identifier
    //
    fn handle_ident_or_string(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Identifier));

        match next_char {
            // If we consume a '"' then we are definitely a string
            // literal.
            '"' => {
                try_action!(self.consume(1));
                self.transition(State::StringLiteral);
            }
            // If we consume an '8' then we could still be consuming
            // an encoding prefix or we could be an identifier (eg
            // "u88")
            '8' => {
                try_action!(self.consume(1));
                self.transition(State::IdentOrStringU8);
            }
            // Otherwise we are an identifier. We will not consume
            // so that all the error checking and next state
            // transitions (eg '+') can be handled by the Identifier
            // state
            _ => {
                self.transition(State::Identifier);
            }
        }

        StateMachineAction::Continue
    }

    // When we enter this state we will have consumed 'u8' which is
    // either an `encoding-prefix` or the start of an identifier.
    //
    // Consumes: "?
    //
    // On EOF: Emit an Identifier
    //
    fn handle_ident_or_string_u8(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Identifier));

        match next_char {
            // If we consume a '"' then we are definitely a string
            // literal.
            '"' => {
                try_action!(self.consume(1));
                self.transition(State::StringLiteral);
            }
            // Otherwise we are an identifier. We will not consume
            // so that all the error checking and next state
            // transitions (eg '+') can be handled by the Identifier
            // state
            _ => {
                self.transition(State::Identifier);
            }
        }
        StateMachineAction::Continue
    }

    // When we enter this state we will have consumed one or more
    // `identifier-nondigit` characters we can then consume all
    // remaining `identifier-nondigit` and `digit` values.
    //
    // Consumes: ([a-zA-Z0-9_]|\u[0-9a-f]{4}|\U[0-9a-f]{8})*
    //
    // On EOF: Emits an identifier
    //
    // [c11]: §6.4.2.1 ¶1
    fn handle_identifier(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Identifier));

        match next_char {
            // This can only begin a UCN so we should peek for the
            // following character. This peek can find an EOF
            // in which case we emit the token and then go back
            // to start because we will treat the next as a newline escape
            '\\' => {
                if let Some((_esc_byte, esc_char)) = self.try_peek() {
                    match esc_char {
                        // Find a 4 byte UCN
                        'u' => {
                            try_action!(self.consume(2));
                            self.transition(State::UCN {
                                to_count: 4,
                                return_to: Box::new(State::Identifier),
                            });
                        }
                        // Find an 8 byte UCN
                        'U' => {
                            try_action!(self.consume(2));
                            self.transition(State::UCN {
                                to_count: 8,
                                return_to: Box::new(State::Identifier),
                            });
                        }
                        c => {
                            self.transition(State::Error(Box::new(State::Identifier)));
                            return StateMachineAction::Emit(Err(Error::UnexpectedCharacter {
                                found: c,
                                expected: vec!['u', 'U'],
                            }));
                        }
                    }
                } else {
                    // Because it is `None` we found EOF so emit an
                    // identifier and go back to start
                    self.transition(State::Start);
                    return self.emit(TokenType::Identifier);
                }
            }

            // Consume an '_' character
            '_' => {
                try_action!(self.consume(1));
            }

            // Consume any digit in [0-9a-zA-Z]
            _ if next_char.is_ascii_alphanumeric() => {
                try_action!(self.consume(1));
            }

            // We must be at the end of the identifier
            // NOTE: Technically the implementation can define other
            // characters to be allowed in an identifier but we don't
            // do that right now.
            _ => {
                self.transition(State::Start);
                return self.emit(TokenType::Identifier);
            }
        }

        StateMachineAction::Continue
    }

    // When we enter this state we will have consumed an optional
    // `encoding-prefix` and an opening '"'. We will continue to
    // consume characters until we find an unescaped close quote.
    //
    // Consumes: (\.|[^\"])*"
    //
    // On EOF: Emit an error about an unterminated string
    //
    // Other Errors: Emit an error if a newline is encountered
    //
    fn handle_string_literal(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_error(Error::StringLiteralMissingClosingQuote));

        match next_char {
            '\r' | '\n' => {
                // This is recoverable if needed (we can pretend to inject a quote)
                // so set the state to the error state and allow it to recover
                self.transition(State::Error(Box::new(State::StringLiteral)));
                return StateMachineAction::Emit(Err(Error::StringLiteralMissingClosingQuote));
            }
            '\\' => {
                // If we have an escape we need to consume another character so we peek
                // again to make sure we don't run off the end of the string
                trace!("Peeking for escape");
                try_action!(self.peek_or_error(Error::StringLiteralMissingClosingQuote));
                try_action!(self.consume(2));
            }

            // On a closing quote we will emit a string literal and return to start
            '"' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                return self.emit(TokenType::StringLiteral);
            }

            // Otherwise we will continue consuming the input characters
            _ => {
                try_action!(self.consume(1));
            }
        }
        StateMachineAction::Continue
    }

    // When we enter this state we will have consumed a '\u' or '\U'
    // and potentially a number of Hex characters. Unlike other
    // states this doesn't actually emit tokens but will consume a
    // UCN and then return to the state which called it.
    //
    // Consumes: [0-9a-fA-F]{to_count}
    //
    // On EOF: Emit an early EOF error
    //
    // [c11]: §6.4.3 ¶1
    fn handle_ucn(&mut self, to_count: usize, return_to: Box<State>) -> StateMachineAction<'s> {
        for i in 0..to_count {
            let next_char = try_action!(self.peek());

            if next_char.is_ascii_hexdigit() {
                try_action!(self.consume(1));
            } else {
                self.transition(State::Error(Box::new(State::UCN {
                    to_count: to_count - i,
                    return_to,
                })));
                return StateMachineAction::Emit(Err(Error::UnexpectedCharacter {
                    found: next_char,
                    expected: vec![
                        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
                        'f',
                    ],
                }));
            }
        }
        self.transition(*return_to);
        StateMachineAction::Continue
    }

    // When we enter this state we will have to have consumed at
    // least one digit so we can consume anything that is part of a
    // PPNumber
    //
    // Consumes:
    fn handle_ppnumber(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::PPNumber));

        match next_char {
            'e' | 'E' | 'p' | 'P' => {
                try_action!(self.consume(1));
                self.transition(State::PPNumberExponent);
            }

            _ if next_char.is_ascii_alphanumeric() => {
                try_action!(self.consume(1));
            }

            '_' | '.' => {
                try_action!(self.consume(1));
            }

            // This can only begin a UCN so we should peek for the
            // following character. This peek can find an EOF
            // in which case we emit the token and then go back
            // to start because we will treat the next as a newline escape
            '\\' => {
                if let Some((_esc_byte, esc_char)) = self.try_peek() {
                    match esc_char {
                        // Find a 4 byte UCN
                        'u' => {
                            try_action!(self.consume(2));
                            self.transition(State::UCN {
                                to_count: 4,
                                return_to: Box::new(State::PPNumber),
                            });
                        }
                        // Find an 8 byte UCN
                        'U' => {
                            try_action!(self.consume(2));
                            self.transition(State::UCN {
                                to_count: 8,
                                return_to: Box::new(State::PPNumber),
                            });
                        }
                        c => {
                            self.transition(State::Error(Box::new(State::PPNumber)));
                            return StateMachineAction::Emit(Err(Error::UnexpectedCharacter {
                                found: c,
                                expected: vec!['u', 'U'],
                            }));
                        }
                    }
                } else {
                    // Because it is `None` we found EOF so emit an
                    // identifier and go back to start
                    self.transition(State::Start);
                    return self.emit(TokenType::PPNumber);
                }
            }

            _ => {
                self.transition(State::Start);
                return self.emit(TokenType::PPNumber);
            }
        }
        StateMachineAction::Continue
    }

    // When we enter this state we will have consumed part of a
    // PPNumber and an 'e' | 'E' | 'p' | 'P'. The only time we can
    // consume a '+' or '-' in a PPNumber is after one of these so
    // if we find that we will consume it.
    // Otherwise we will just go back to the PPNumber state
    //
    // Consumes: (+|-)?
    //
    // On EOF: Emit a PPNumber
    //
    fn handle_ppnumber_exponent(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::PPNumber));

        match next_char {
            '-' | '+' => {
                try_action!(self.consume(1));
            }
            _ => {}
        }
        self.transition(State::PPNumber);
        StateMachineAction::Continue
    }

    // When we enter this state we will have seen one '.' and we have
    // to check for either a digit (PPNumber) or two '..' so we will
    // can transfer to a PPNumber or emit a '.' or a '...'
    //
    // Consumes: (..|Digit)?
    //
    // On EOF: Emits a punctuator
    //
    fn handle_ppnumber_or_dot(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            _ if next_char.is_ascii_digit() => {
                try_action!(self.consume(1));
                self.transition(State::PPNumber);
            }

            '.' => {
                if let Some((_, esc_char)) = self.try_peek() {
                    match esc_char {
                        '.' => {
                            try_action!(self.consume(2));
                            self.transition(State::Start);
                            return self.emit(TokenType::Punctuator);
                        }
                        _ => {
                            self.transition(State::Start);
                            return self.emit(TokenType::Punctuator);
                        }
                    }
                } else {
                    self.transition(State::Start);
                    return self.emit(TokenType::Punctuator);
                }
            }

            _ => {
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }
        }
        StateMachineAction::Continue
    }

    // When we reach this state we will have consumed a '>'. We then
    // search for other symbols which can make a valid punct token
    //
    // Consumes: (> | =)?
    //
    // On Eof: Emits a Punctuator token
    //
    fn handle_punct_gt(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            '=' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }

            '>' => {
                try_action!(self.consume(1));
                self.transition(State::PunctAndEqual);
            }

            _ => {
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }
        }
        StateMachineAction::Continue
    }

    // When we reach this state we will have consumed a '<'. We then
    // search for other symbols which can make a valid punct token
    //
    // Consumes: (< | = | : | %)?
    //
    // On Eof: Emits a Punctuator token
    //
    fn handle_punct_lt(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            '=' | ':' | '%' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }

            '<' => {
                try_action!(self.consume(1));
                self.transition(State::PunctAndEqual);
            }

            _ => {
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }
        }
        StateMachineAction::Continue
    }

    // When we reach this state we will have consumed a punctuation character
    // which can also be combined with an '=' to form a valid punctuator.
    // Here we will consume an '=' or emit the token as is.
    //
    // Consumes: =?
    //
    // On Eof: Emits a punctuator
    //
    fn handle_punct_and_equal(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            '=' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                self.emit(TokenType::Punctuator)
            }

            _ => {
                self.transition(State::Start);
                self.emit(TokenType::Punctuator)
            }
        }
    }

    // When we enter this state we have consumed a '%' so we can consume a number
    // of other tokens to make valid punctuators
    //
    // Consumes: (=| > | :)?
    //
    // On EOF: Emits a punctuator
    //
    fn handle_punct_percent(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            '=' | '>' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                self.emit(TokenType::Punctuator)
            }

            ':' => {
                try_action!(self.consume(1));
                self.transition(State::AltPound);
                StateMachineAction::Continue
            }

            _ => {
                self.transition(State::Start);
                self.emit(TokenType::Punctuator)
            }
        }
    }

    // When we enter this state we have consumed a ':' so we can consume a number
    // of other tokens to make valid punctuators
    //
    // Consumes: >?
    //
    // On EOF: Emits a punctuator
    //
    fn handle_punct_colon(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            '>' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                self.emit(TokenType::Punctuator)
            }

            _ => {
                self.transition(State::Start);
                self.emit(TokenType::Punctuator)
            }
        }
    }

    // When we reach this state we have consumed a '%:' we need to see if we
    // have on alternate pound or two so we need to peek ahead two tokens
    //
    // Consumes: (%:)?
    //
    // On EOF: Issues a Punctuation
    //
    fn handle_alt_pound(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        if next_char == '%' {
            if let Some((_, peek_char)) = self.try_peek() {
                if peek_char == ':' {
                    self.transition(State::Start);
                    try_action!(self.consume(2));
                    return self.emit(TokenType::Punctuator);
                }
            }
        }
        self.transition(State::Start);
        self.emit(TokenType::Punctuator)
    }

    // When we reach this state we will have consumed a '-'. We are only going
    // to check if we produce the "->" token othewise we will defer to the
    // PunctDoulbeOrEqual state
    //
    // Consumes: >?
    //
    // On EOF: Emits a Punctuator
    //
    fn handle_punct_minus(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        if next_char == '>' {
            try_action!(self.consume(1));
            self.transition(State::Start);
            self.emit(TokenType::Punctuator)
        } else {
            self.transition(State::PunctDoubleOrEq('-'));
            StateMachineAction::Continue
        }
    }

    // When we reach this state we will have consume a '<char>' we can then consume
    // another '<char>' or an '='
    //
    // Consumes: (<char> | =)?
    //
    // On EOF: Emits a Punctuator
    //
    fn handle_punct_double_or_equal(&mut self, orig: char) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        self.transition(State::Start);
        match next_char {
            '=' => {
                try_action!(self.consume(1));
                self.emit(TokenType::Punctuator)
            }
            _ if next_char == orig => {
                try_action!(self.consume(1));
                self.emit(TokenType::Punctuator)
            }
            _ => self.emit(TokenType::Punctuator),
        }
    }

    // When we reach this state we will have consume a '<char>' we can then consume
    // another '<char>'
    //
    // Consumes: (<char>)?
    //
    // On EOF: Emits a Punctuator
    //
    fn handle_punct_double(&mut self, orig: char) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        self.transition(State::Start);
        match next_char {
            _ if next_char == orig => {
                try_action!(self.consume(1));
                self.emit(TokenType::Punctuator)
            }
            _ => self.emit(TokenType::Punctuator),
        }
    }

    // When we enter this space we will have consumed some non-newline whitespace
    // character.
    //
    // Consumes: <non-newline whitespace>*
    //
    // On EOF: Emit Whitespace
    //
    fn handle_whitespace(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Whitespace));

        match next_char {
            '\n' | '\r' => {
                self.transition(State::Start);
                self.emit(TokenType::Whitespace)
            }

            _ if next_char.is_ascii_whitespace() => {
                try_action!(self.consume(1));
                StateMachineAction::Continue
            }

            _ => {
                self.transition(State::Start);
                self.emit(TokenType::Whitespace)
            }
        }
    }

    // When we enter this state we will have seen a '\n' or '\r'
    //
    // Consumes: (\n\r|\r\n|\n|\r)
    //
    // On EOF: Emits an EOL token
    //
    fn handle_eol(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek());
        let sec_char = try_action!(self.peek_or_emit(TokenType::EOL));

        match (next_char, sec_char) {
            ('\n', '\r') | ('\r', '\n') => {
                try_action!(self.consume(2));
                self.transition(State::Start);
                self.emit(TokenType::EOL)
            }

            ('\n', _) | ('\r', _) => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                self.emit(TokenType::EOL)
            }

            (_, _) => {
                unreachable!();
            }
        }
    }

    // When we get here we will have consumed a '/' and it can either be a
    // division or the start of a comment.
    //
    // Consumes (=|/|*)?
    //
    // On EOF: Emits a Punctuator
    //
    fn handle_divide_or_comment(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Punctuator));

        match next_char {
            '/' => {
                try_action!(self.consume(1));
                self.transition(State::LineComment);
            }

            '*' => {
                try_action!(self.consume(1));
                self.transition(State::BlockComment);
            }

            '=' => {
                try_action!(self.consume(1));
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }

            _ => {
                self.transition(State::Start);
                return self.emit(TokenType::Punctuator);
            }
        }

        StateMachineAction::Continue
    }

    // When we get to this state we will have consumed "//"
    //
    // Consumed: [^\n]*
    //
    // On EOF: Emit a Comment
    //
    fn handle_line_comment(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek_or_emit(TokenType::Comment));

        if next_char == '\n' {
            self.transition(State::Start);
            self.emit(TokenType::Comment)
        } else {
            self.consume(1);
            StateMachineAction::Continue
        }
    }

    // When we get to this state we will have consumed '/*'
    //
    // Consumed: .*?*/
    //
    // On EOF: Emit an error
    //
    fn handle_block_comment(&mut self) -> StateMachineAction<'s> {
        let next_char = try_action!(self.peek());
        let sec_char = try_action!(self.peek());

        match (next_char, sec_char) {
            ('*', '/') => {
                try_action!(self.consume(2));
                self.transition(State::Start);
                self.emit(TokenType::Comment)
            }

            (_, _) => {
                try_action!(self.consume(1));
                StateMachineAction::Continue
            }
        }
    }

    // When we get here we will be partially through some token, we don't know
    // which token only the state it was in. Most errors we don't recover from
    // right now.
    //
    // We will try to recover if we errored during a StringLiteral and will omit
    // whatever we have seen regardless of if the string is properly terminated
    //
    fn handle_error(&mut self, other_state: State) -> StateMachineAction<'s> {
        match other_state {
            State::StringLiteral => {
                try_action!(self.peek_or_emit(TokenType::StringLiteral));
                self.transition(State::Start);
                self.emit(TokenType::StringLiteral)
            }
            _ => {
                // If we are iterating stop here
                self.hit_eof = true;
                StateMachineAction::Emit(Err(Error::CannotRecoverFromError))
            }
        }
    }

    fn peek(&mut self) -> ActionResult<'s, char> {
        self.peek_or_error(Error::EarlyEOF)
    }

    fn peek_or_error(&mut self, error: Error) -> ActionResult<'s, char> {
        match self.try_peek() {
            Some(value) => ActionResult::Ok(value.1),
            None => {
                self.transition(State::Error(Box::new(self.state.clone())));
                ActionResult::Err(error)
            }
        }
    }

    fn peek_or_emit(&mut self, token_type: TokenType) -> ActionResult<'s, char> {
        match self.try_peek() {
            Some(value) => ActionResult::Ok(value.1),
            None => {
                self.transition(State::Start);
                ActionResult::Token(self.build_token(token_type))
            }
        }
    }

    fn try_peek(&mut self) -> Option<(usize, char)> {
        self.char_input.peek()
    }

    fn reset_peek(&mut self) {
        self.char_input.reset_peek();
    }

    #[cfg_attr(tarpaulin, skip)]
    fn consume(&mut self, count: usize) -> ActionResult<'s, ()> {
        for _ in 0..count {
            match self.char_input.consume() {
                Ok(()) => {}
                Err(()) => {
                    return ActionResult::Err(Error::ConsumedTooManyCharacters);
                }
            };
        }
        ActionResult::Ok(())
    }

    // /// Insert a character into the stream. This can be used by the preprocessor
    // /// to avoid an error.
    // pub fn insert_character(&mut self, c: char) {
    //     assert!(self.buffer_idx == 0);
    //     let b = self
    //         .advance_peek()
    //         .map(|(b, _)| b)
    //         .unwrap_or(self.string_end_byte);
    //     self.return_peek();

    //     self.buffer.push_front((b, c));
    // }

    fn transition(&mut self, state: State) {
        trace!("Transition to: {:?}", state);
        self.state = state;
    }

    fn emit(&mut self, token_type: TokenType) -> StateMachineAction<'s> {
        StateMachineAction::Emit(Ok(self.build_token(token_type)))
    }

    fn build_token(&mut self, token_type: TokenType) -> Token<'s> {
        match token_type {
            TokenType::EOF => {
                self.hit_eof = true;
                Token {
                    token_type,
                    start_byte: self.char_input.get_len(),
                    end_byte: self.char_input.get_len(),
                    text: Cow::Owned("".into()),
                }
            }
            _ => {
                let (start_byte, end_byte) = self.char_input.get_consumed_range();
                Token {
                    token_type,
                    start_byte,
                    end_byte,
                    text: self.char_input.get_consumed(),
                }
            }
        }
    }

    /// The tokenizer is not designed to know about the pre-processing state.
    /// Because of this it needs to be told if it is allowed to tokenize a
    /// header name. If this is not set all quotes will indicate a quoted string
    /// not a header name.
    ///
    /// This also allows the preprocessor to enable header name tokens in any
    /// location it wants. For example some `#pragma` definitions might want
    /// to allow header name tokens.
    pub fn set_header_name_allowed(&mut self, is_allowed: bool) {
        self.is_header_valid = is_allowed;
    }
}

#[cfg_attr(tarpaulin, skip)]
impl<'a, S: CharSource<'a>> Iterator for Tokenizer<'a, S> {
    type Item = Result<Token<'a>, Error>;
    /// Iterates over all the tokens in the stream. The iterator will return
    /// `None` after either `EOF` or an `UnrecoverableError` is produced.
    fn next(&mut self) -> Option<Self::Item> {
        if self.hit_eof {
            None
        } else {
            Some(self.get_next_token())
        }
    }
}

enum StateMachineAction<'s> {
    Continue,
    Emit(Result<Token<'s>, Error>),
}

enum ActionResult<'s, T> {
    Ok(T),
    Err(Error),
    Token(Token<'s>),
}

#[derive(Debug, PartialEq, Eq, Clone)]
enum State {
    Start,

    Whitespace,
    EOL,

    LineComment,
    BlockComment,
    DivideOrComment,

    QuotedHeaderName,
    BracketedHeaderName,

    IdentOrString,
    IdentOrStringU8,
    StringLiteral,

    Identifier,
    UCN {
        to_count: usize,
        return_to: Box<State>,
    },

    PPNumberOrDot,
    PPNumber,
    PPNumberExponent,

    PunctGTStart,
    PunctLTStart,
    PunctAndEqual,
    PunctPercent,
    PunctColon,
    PunctMinus,
    PunctDouble(char),
    PunctDoubleOrEq(char),
    AltPound,
    Error(Box<State>),
}

/// Defines the token type from the preprocesor token string
/// Every character (except escaped newlines) will be turned into a token of
/// one of the following types
//[c11]: S6.4 Lexical elements
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum TokenType {
    /// An identifier
    Identifier,
    /// A Preprocessor number
    PPNumber,
    /// A Quoted or Angle-braced string which references a header
    HeaderName,
    /// A single character constant
    CharacterConstant,
    /// A quoted string
    StringLiteral,
    /// A single punctuator
    Punctuator,
    /// Non-newline whitespace token
    Whitespace,
    /// A line-ending character.
    EOL,
    /// A token that indicates we reached the end of the file
    EOF,
    /// A token that indicates a comment
    Comment,
}

#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Token<'s> {
    pub token_type: TokenType,

    pub start_byte: usize,
    pub end_byte: usize,

    pub text: Cow<'s, str>,
}

#[derive(Debug, PartialEq, Eq)]
pub enum Error {
    EarlyEOF,
    StringLiteralMissingClosingQuote,
    InvalidIdentifierCharacter(char, usize),
    UnexpectedCharacter { found: char, expected: Vec<char> },
    ConsumedTooManyCharacters,
    CannotRecoverFromError,
}