kconfig-parser 0.1.1

/*
 Cargo KConfig - KConfig parser
 Copyright (C) 2022  Sjoerd van Leent

--------------------------------------------------------------------------------

Copyright Notice: Apache

Licensed under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of the
License at

   https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.

--------------------------------------------------------------------------------

Copyright Notice: GPLv2

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.

--------------------------------------------------------------------------------

Copyright Notice: MIT

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the “Software”), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

//! This file contains the lexer and the logic of the lexer, implementing
//! the lexer machine.

use std::io::{BufRead, BufReader, Read};

use crate::lex::structs::EqualityOperator;

use super::{
    structs::{Keyword, Lexicon, Token},
    LexerBase,
};

fn maybe_keyword(s: &str) -> Option<Keyword> {
    match s {
        "source" => Some(Keyword::Source),
        "mainmenu" => Some(Keyword::Mainmenu),
        "config" => Some(Keyword::Config),
        "menuconfig" => Some(Keyword::Menuconfig),
        "choice" => Some(Keyword::Choice),
        "endchoice" => Some(Keyword::Endchoice),
        "menu" => Some(Keyword::Menu),
        "endmenu" => Some(Keyword::Endmenu),
        "if" => Some(Keyword::If),
        "endif" => Some(Keyword::Endif),
        "bool" => Some(Keyword::Bool),
        "def_bool" => Some(Keyword::DefBool),
        "tristate" => Some(Keyword::Tristate),
        "def_tristate" => Some(Keyword::DefTristate),
        "string" => Some(Keyword::String),
        "hex" => Some(Keyword::Hex),
        "int" => Some(Keyword::Int),
        "default" => Some(Keyword::Default),
        "depends" => Some(Keyword::Depends),
        "on" => Some(Keyword::On),
        "select" => Some(Keyword::Select),
        "imply" => Some(Keyword::Imply),
        "visible" => Some(Keyword::Visible),
        "range" => Some(Keyword::Range),
        "prompt" => Some(Keyword::Prompt),
        "comment" => Some(Keyword::Comment),
        _ => None,
    }
}

/// Trims help lines, removes whitespace which is in front and/or back
/// of any given help line, and concatenates the lines with a new line
/// symbol. Then returns the complete set of lines, trimmed at the
/// front of the complete text, keeping white lines within the middle
/// of the text (used for paragraph separation.)
fn trim_help(s: &str) -> String {
    let mut result = String::new();
    for line in s.lines() {
        let trimmed = line.trim();
        result.push_str(trimmed);
        result.push('\n');
    }
    result.trim().to_string()
}

/// Strips the last character from the given string, and returns
/// a new String without that character.
fn rstrip(s: &str) -> String {
    s[..s.len() - 1].to_string()
}

/// This structure holds the data of the actual lexer. It is expected
/// that this structure can be mutated by the `next_token` function,
/// which advances to the next token, and returns the term found by
/// the lexer.
pub struct Lexer<T>
where
    T: Read,
{
    reader: BufReader<T>,
    reading_line: Option<String>,
    back: Option<char>,
    open_macro: usize,
    col: usize,
    line: usize,
}

/// The lexer implementation allows to read the tokens from a given
/// type implementing the Read trait. If an error occurs while reading,
/// or EOT has been reached, the `next_token` function will return
/// Error or EOT as its term.
impl<T: Read> Lexer<T> {
    /// Creates a new lexer from the given input, where input
    /// supports the Read trait.
    pub fn create(input: T) -> Self {
        return Self {
            reader: BufReader::new(input),
            reading_line: None,
            back: None,
            open_macro: 0,
            col: 0,
            line: 1,
        };
    }

    /// Finds the next character of the input, if available. If
    /// available, the characters is returned as Some Option in
    /// a Result. If the characters are depleted, then None will
    /// be returned in a Result. If an I/O Error occurred, it will
    /// be returned in the Result.
    fn next_char(&mut self) -> std::io::Result<Option<char>> {
        // If a character has been pushed back (can only occur once),
        // return that character.
        if let Some(c) = self.back {
            self.back = None;
            return Ok(Some(c));
        }

        if self.reading_line == None {
            let mut new_line: String = String::new();
            match self.reader.read_line(&mut new_line)? {
                0 => (),
                _ => self.reading_line = Some(new_line),
            }
        }

        // If it is still None, then nothing has been read,
        // this must be the end.
        if let Some(s) = &self.reading_line {
            // There will be a next character
            let c = s.chars().nth(0).unwrap();
            let new_string = s[1..].to_string();
            match new_string.len() > 0 {
                true => self.reading_line = Some(new_string),
                false => self.reading_line = None,
            }
            Ok(Some(c))
        } else {
            Ok(None)
        }
    }

    /// Pushes the last character back into the Lexer as this character
    /// is not understood by the current token, but might be valid for
    /// the next token. Typically, this will occur at the end of parsing
    /// the current token.
    fn push_back(&mut self, c: char) {
        self.back = Some(c);
        if self.col > 0 {
            self.col -= 1;
        }

        if c == '\n' {
            self.line -= 1;
        }
    }
}

impl<T: Read> LexerBase for Lexer<T> {
    /// Finds the next token in a mutable self. If the next token
    /// can be found, returns it into Token. If the file or stream
    /// of data is found, the special term EOT is returned. If an
    /// error occurred of some sort, the special term Error is
    /// returned.
    fn next_token(&mut self) -> Token {
        // If whitespace is found before a token, swallow it

        enum Machine {
            Start,
            MacroStart,
            Ident,
            Immediate,
            Append,
            String,
            Lt,
            Gt,
            Eq,
            Ne,
            And,
            Or,

            // Help is weird, the way it needs to be parsed is rather destructive
            Help,
            ProperHelp,
            // Currently running inside a macro
            Macro,
            MacroIdent,

            // Deals with a comment line
            MacroComment,
            Comment,
        }

        // Set to true if an escape symbol has been found, and the next symbol
        // should be interpreted as a char (where available).
        let mut escape = false;

        // When parsing a help section, is set to the indent of the current help
        // token. Each line with a higher indent is interpreted as help text,
        // otherwise it will revert to 'regular' tokenization.
        let mut exp_help_indent = 0;

        // The current text being parsed, which can be part of the resulting
        // token.
        let mut cs = String::default();

        // The current column number
        let mut column = self.col;

        // The current line number
        let mut line = self.line;

        // When a comment has been opened, this is set and alters the way the
        // parsing works until the comment is closed (typically by a newline symbol)
        let mut open_comment = false;

        // When the previous token either opened or remained in macro mode, the machine
        // is set to Macro, otherwise to Start. Note that because macro mode can be
        // nested, this is reflected in the open_macro numeric counter.
        let mut m = match self.open_macro > 0 {
            false => Machine::Start,
            true => Machine::Macro,
        };

        // Loops and 'eats' every single character until a token has been
        // properly read, EOT is reached or an Error is returned.
        loop {
            let result = self.next_char();
            match result {
                Ok(maybe) => match maybe {
                    Some(c) => {
                        self.col += 1;
                        if c == '\n' {
                            self.line += 1;
                            self.col = 0;
                        } else if c == '\r' {
                            self.col -= 1;
                            continue;
                        }

                        if (c != '#' || escape) && !open_comment {
                            cs.push(c)
                        }
                        match m {
                            // If in open_macro mode, there are only identifiers,
                            // unless a comma is specified, another macro start is
                            // identified or a close is identified.
                            Machine::Macro => match c {
                                ',' => return Token::create(Lexicon::Comma, column, line, &cs),
                                ')' => {
                                    self.open_macro -= 1;
                                    return Token::create(Lexicon::Close, column, line, &cs);
                                }
                                '$' => m = Machine::MacroStart,
                                '#' => {
                                    m = Machine::MacroComment;
                                    open_comment = true;
                                }
                                '\\' => {
                                    escape = true;
                                    m = Machine::MacroIdent;
                                    cs = String::new();
                                }
                                '(' => {
                                    return Token::create(
                                        Lexicon::Error(cs.clone()),
                                        column,
                                        line,
                                        &cs,
                                    );
                                }
                                _ => m = Machine::MacroIdent,
                            },
                            Machine::MacroIdent => match c {
                                '(' => {
                                    if !escape {
                                        return Token::create(
                                            Lexicon::Error(cs.clone()),
                                            column,
                                            line,
                                            &cs,
                                        );
                                    }
                                    escape = false;
                                }
                                '$' | ',' | ')' => {
                                    if !escape {
                                        self.push_back(c);
                                        let s = rstrip(&cs);
                                        return Token::create(
                                            Lexicon::Identifier(s),
                                            column,
                                            line,
                                            &cs,
                                        );
                                    }
                                    escape = false;
                                }
                                '\\' => {
                                    escape = !escape;
                                    if escape {
                                        cs = cs[..cs.len() - 1].to_string();
                                    }
                                }
                                '#' => {
                                    if !escape {
                                        self.push_back(c);
                                        open_comment = true;
                                        m = Machine::MacroComment;
                                    }
                                    escape = false;
                                }
                                _ => escape = false,
                            },
                            Machine::MacroComment => match c {
                                '\n' => {
                                    m = if cs.len() == 0 {
                                        Machine::Macro
                                    } else {
                                        Machine::MacroIdent
                                    };
                                    open_comment = false;
                                }
                                _ => (),
                            },
                            Machine::Comment => match c {
                                '\n' => {
                                    cs = String::new();
                                    m = Machine::Start;
                                    line = self.line;
                                    column = self.col;
                                    open_comment = false;
                                }
                                _ => (),
                            },
                            Machine::Start => match c {
                                // Skip control characters, they don't make sense
                                '\u{0}'..='\u{0D}' | ' ' => {
                                    cs = String::new();
                                    line = self.line;
                                    column = self.col;
                                }
                                // If the token is a comma, token data should be returned likewise
                                '$' => m = Machine::MacroStart,
                                '(' => return Token::create(Lexicon::Open, column, line, &cs),
                                ')' => return Token::create(Lexicon::Close, column, line, &cs),
                                ':' => m = Machine::Immediate,
                                '+' => m = Machine::Append,
                                'a'..='z' | 'A'..='Z' | '0'..='9' | '-' => m = Machine::Ident,
                                '#' => {
                                    open_comment = true;
                                    m = Machine::Comment;
                                }
                                '"' => m = Machine::String,
                                '<' => m = Machine::Lt,
                                '>' => m = Machine::Gt,
                                '=' => m = Machine::Eq,
                                '!' => m = Machine::Ne,
                                '&' => m = Machine::And,
                                '|' => m = Machine::Or,
                                _ => return Token::create_error(column, line, &cs),
                            },
                            Machine::Lt => match c {
                                '=' => {
                                    return Token::create(
                                        Lexicon::EqualityOperator(EqualityOperator::Lte),
                                        column,
                                        line,
                                        &cs,
                                    )
                                }
                                _ => {
                                    self.push_back(c);
                                    return Token::create(
                                        Lexicon::EqualityOperator(EqualityOperator::Lt),
                                        column,
                                        line,
                                        &cs,
                                    );
                                }
                            },
                            Machine::Gt => match c {
                                '=' => {
                                    return Token::create(
                                        Lexicon::EqualityOperator(EqualityOperator::Gte),
                                        column,
                                        line,
                                        &cs,
                                    )
                                }
                                _ => {
                                    self.push_back(c);
                                    return Token::create(
                                        Lexicon::EqualityOperator(EqualityOperator::Gt),
                                        column,
                                        line,
                                        &cs,
                                    );
                                }
                            },
                            Machine::Eq => match c {
                                '=' => {
                                    return Token::create(
                                        Lexicon::EqualityOperator(EqualityOperator::Eq),
                                        column,
                                        line,
                                        &cs,
                                    )
                                }
                                _ => {
                                    self.push_back(c);
                                    return Token::create(Lexicon::Assignment, column, line, &cs);
                                }
                            },
                            Machine::Ne => match c {
                                '=' => {
                                    return Token::create(
                                        Lexicon::EqualityOperator(EqualityOperator::Ne),
                                        column,
                                        line,
                                        &cs,
                                    )
                                }
                                _ => {
                                    self.push_back(c);
                                    return Token::create(Lexicon::Not, column, line, &cs);
                                }
                            },
                            Machine::And => match c {
                                '&' => return Token::create(Lexicon::And, column, line, &cs),
                                _ => {
                                    self.push_back(c);
                                    let s = rstrip(&cs);
                                    return Token::create(Lexicon::Error(s), column, line, &cs);
                                }
                            },
                            Machine::Or => match c {
                                '|' => return Token::create(Lexicon::Or, column, line, &cs),
                                _ => {
                                    self.push_back(c);
                                    let s = rstrip(&cs);
                                    return Token::create(Lexicon::Error(s), column, line, &cs);
                                }
                            },
                            Machine::String => match c {
                                '\\' => escape = !escape,
                                '"' => {
                                    if !escape {
                                        return Token::create(
                                            Lexicon::String(cs.to_string()),
                                            column,
                                            line,
                                            &cs,
                                        );
                                    }
                                    escape = false;
                                }
                                _ => escape = false,
                            },
                            Machine::MacroStart => match c {
                                // The next token of a macro should be (
                                '(' => {
                                    self.open_macro += 1;
                                    return Token::create(Lexicon::MacroOpen, column, line, &cs);
                                }
                                _ => {
                                    self.push_back(c);
                                    let s = rstrip(&cs);
                                    return Token::create_error(column, line, &s);
                                }
                            },

                            Machine::Immediate => match c {
                                '=' => {
                                    return Token::create(
                                        Lexicon::ImmediateAssignment,
                                        column,
                                        line,
                                        &cs,
                                    )
                                }
                                _ => {
                                    self.push_back(c);
                                    let s = rstrip(&cs);
                                    return Token::create_error(column, line, &s);
                                }
                            },

                            Machine::Append => match c {
                                '=' => {
                                    return Token::create(
                                        Lexicon::AppendAssignment,
                                        column,
                                        line,
                                        &cs,
                                    )
                                }
                                _ => {
                                    self.push_back(c);
                                    let s = rstrip(&cs);
                                    return Token::create_error(column, line, &s);
                                }
                            },

                            Machine::Ident => match c {
                                // The next token should be within range 'a' .. 'z',
                                // 'A' .. 'Z', '0' .. '9', '-', '_',
                                'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' => m = Machine::Ident,
                                _ => {
                                    self.push_back(c);
                                    let s = rstrip(&cs);
                                    match maybe_keyword(&s) {
                                        Some(k) => {
                                            return Token::create(
                                                Lexicon::Keyword(k),
                                                column,
                                                line,
                                                &cs,
                                            )
                                        }
                                        None => {
                                            if s.eq("help") {
                                                m = Machine::Help;
                                                exp_help_indent = column + 1;
                                                cs = String::new();
                                            } else {
                                                return Token::create(
                                                    Lexicon::Identifier(s),
                                                    column,
                                                    line,
                                                    &cs,
                                                );
                                            }
                                        }
                                    }
                                }
                            },
                            Machine::Help => match c {
                                // This is the weirdest token, it consists of an identifier Help, then
                                // followed by a help text, which can not be parsed, other than the help
                                // needs to be on the next line, and the next line needs to start further
                                // indented from the current line.
                                '\n' => {
                                    m = Machine::ProperHelp;
                                    open_comment = false;
                                }
                                '\u{0}'..='\u{0D}' => cs = String::new(),
                                ' ' => cs = String::new(),
                                '#' => open_comment = true,
                                _ => {
                                    if !open_comment {
                                        return Token::create_error(self.col, line, &cs);
                                    }
                                }
                            },
                            Machine::ProperHelp => {
                                if self.col <= exp_help_indent {
                                    match c {
                                        '\n' => (),
                                        '\u{0}'..='\u{0D}' => cs = rstrip(&cs),
                                        ' ' => cs = rstrip(&cs),
                                        _ => {
                                            self.push_back(c);
                                            let s = rstrip(&cs);
                                            let help_str = trim_help(&s);
                                            if help_str.len() == 0 {
                                                // If no help text was found, then it is an error
                                                return Token::create(
                                                    Lexicon::Error(s),
                                                    column,
                                                    line,
                                                    &cs,
                                                );
                                            } else {
                                                return Token::create(
                                                    Lexicon::Help(trim_help(&s)),
                                                    self.col,
                                                    line,
                                                    &cs,
                                                );
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                    // If no data has been retrieved anymore, this is EOT if in the start state of the machine,
                    // otherwise it could be an identifier
                    None => {
                        return match m {
                            // Although ending when finding a Macro Identifier makes no sense to the AST, it is
                            // validly lexed. Let the AST worry about the issue.
                            Machine::MacroIdent => {
                                let s = cs.to_string();
                                return Token::create(Lexicon::Identifier(s), column, line, &cs);
                            }
                            Machine::Ident => match maybe_keyword(&cs) {
                                Some(k) => {
                                    return Token::create(Lexicon::Keyword(k), column, line, &cs)
                                }
                                None => {
                                    let s = cs.to_string();
                                    if s.eq("help") {
                                        // In this case, a help keyword followed by EOT is an error and makes no
                                        // sense.
                                        return Token::create(Lexicon::Error(s), column, line, &cs);
                                    } else {
                                        return Token::create(
                                            Lexicon::Identifier(s),
                                            column,
                                            line,
                                            &cs,
                                        );
                                    }
                                }
                            },
                            Machine::ProperHelp => {
                                Token::create(Lexicon::Help(trim_help(&cs)), self.col, line, &cs)
                            }

                            // Eq can make a valid EOT if only assignment was set
                            Machine::Eq => Token::create(Lexicon::Assignment, self.col, line, &cs),

                            // Ne can make a valid EOT if only not was set
                            Machine::Ne => Token::create(Lexicon::Not, self.col, line, &cs),

                            // Any of these states can create a valid EOT
                            Machine::Start | Machine::Comment | Machine::Macro => {
                                Token::create_eot(self.col, line)
                            }

                            // Any other state generates an error. The next call will then create an EOT
                            _ => Token::create(Lexicon::Error(cs.to_string()), column, line, &cs),
                        };
                    }
                },
                // If File I/O error occures, consider it also EOT
                _ => return Token::create_eot(self.col, line),
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::super::LexerBase;
    use super::*;

    #[test]
    fn test_keywords() {
        let s = &mut "source mainmenu config menuconfig choice endchoice menu endmenu if endif bool def_bool \
                      tristate def_tristate string hex int default depends on select imply visible range prompt comment".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Keyword(Keyword::Source), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Mainmenu), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Config), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Menuconfig), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Choice), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Endchoice), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Menu), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Endmenu), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::If), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Endif), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Bool), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::DefBool), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Tristate), l.next_token().term());
        assert_eq!(
            Lexicon::Keyword(Keyword::DefTristate),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Keyword(Keyword::String), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Hex), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Int), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Default), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Depends), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::On), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Select), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Imply), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Visible), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Range), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Prompt), l.next_token().term());
        assert_eq!(Lexicon::Keyword(Keyword::Comment), l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_identifier() {
        let s = &mut "foo BAR".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::Identifier("BAR".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_string() {
        let s = &mut "\"Hello \\\" World\"".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::String("\"Hello \\\" World\"".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_assignment() {
        let s = &mut "=".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Assignment, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_assignment_with_space() {
        let s = &mut "= ".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Assignment, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_immediate_assignment() {
        let s = &mut ":=".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::ImmediateAssignment, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_append_assignment() {
        let s = &mut "+=".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::AppendAssignment, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_eq_operators() {
        let s = &mut "< > <= >= == !=".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::EqualityOperator(EqualityOperator::Lt),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::EqualityOperator(EqualityOperator::Gt),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::EqualityOperator(EqualityOperator::Lte),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::EqualityOperator(EqualityOperator::Gte),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::EqualityOperator(EqualityOperator::Eq),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::EqualityOperator(EqualityOperator::Ne),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_binary_comparison() {
        let s = &mut "&& ||".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::And, l.next_token().term());
        assert_eq!(Lexicon::Or, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_not() {
        let s = &mut "!".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Not, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_not_with_space() {
        let s = &mut "! ".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Not, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_open() {
        let s = &mut "(".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Open, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_close() {
        let s = &mut ")".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_open() {
        let s = &mut "$(".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_ident_comma() {
        let s = &mut "$(foo,bar)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Comma, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("bar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_ident_comma_spaces() {
        let s = &mut "$(foo , bar)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("foo ".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Comma, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier(" bar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_ident_no_keyword() {
        let s = &mut "$(config)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("config".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_comma() {
        let s = &mut "$(,)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(Lexicon::Comma, l.next_token().term());
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_comment() {
        let s = &mut "foo # Foo Bar\nbar".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
        assert_eq!(
            Lexicon::Identifier("bar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_comment() {
        let s = &mut "$(# Foo Bar\n)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    /**
     * This test not only tests comments within macros, but also
     * tests if the identifier is properly surrounding the comment.
     */
    #[test]
    fn test_macro_intermediate_comment() {
        let s = &mut "$(foo # Foo Bar\nbar)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("foo bar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    /**
     * This test if a macro comment and comma before the comment
     * dutifully creates two identifiers
     */
    #[test]
    fn test_macro_comment_with_comma_before() {
        let s = &mut "$(foo, # Foo Bar\nbar)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Comma, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier(" bar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_macro_comment_with_comma_after() {
        let s = &mut "$(foo# Foo Bar\n, bar)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Comma, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier(" bar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_return_character_does_nothing() {
        let s = &mut "foo\rbar".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Identifier("foobar".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_negative_integer() {
        let s = &mut "-123".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Identifier("-123".to_owned()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_help() {
        let s = &mut "help\n  spam ham eggs".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Help("spam ham eggs".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_multiline_help() {
        let s = &mut "help\n  spam\n  ham\n  eggs".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Help("spam\nham\neggs".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_help_with_other_stuff() {
        let s = &mut "help\n  spam ham eggs\nfoo".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Help("spam ham eggs".to_string()),
            l.next_token().term()
        );
    }

    #[test]
    fn test_multiline_help_with_other_stuff() {
        let s = &mut "help\n  spam\n  ham\n  eggs\nfoo".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Help("spam\nham\neggs".to_string()),
            l.next_token().term()
        );
    }

    #[test]
    fn test_help_with_comment() {
        let s = &mut "help # silly comment\n  spam ham eggs".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Help("spam ham eggs".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn start_macro_in_macro() {
        let s = &mut "$(hello$(world))".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("hello".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("world".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }
    #[test]
    fn test_escape_in_macro_identifier() {
        let s = &mut "$(hello\\$\\(world\\))".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("hello$(world)".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }
    #[test]
    fn test_escape_in_macro_start() {
        let s = &mut "$(\\$\\(world\\))".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("$(world)".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }
    #[test]
    fn test_escape_escape_in_macro() {
        let s = &mut "$(\\\\)".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(Lexicon::Identifier("\\".to_string()), l.next_token().term());
        assert_eq!(Lexicon::Close, l.next_token().term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }
    #[test]
    fn test_skip_control_characters() {
        let s = &mut "\tfoo".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_col() {
        let s = &mut "foo bar".as_bytes();
        let mut l = Lexer::create(s);
        let t1 = l.next_token();
        let t2 = l.next_token();
        let t3 = l.next_token();

        assert_eq!(0, t1.column());
        assert_eq!(4, t2.column());
        assert_eq!(7, t3.column());
    }

    #[test]
    fn test_line() {
        let s = &mut "\nfoo\nbar".as_bytes();
        let mut l = Lexer::create(s);
        let t1 = l.next_token();
        let t2 = l.next_token();

        assert_eq!(2, t1.line());
        assert_eq!(3, t2.line());
    }

    #[test]
    fn test_nonsense() {
        let s = &mut "/".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("/".to_string()), t.term());
        assert_eq!(Lexicon::EOT, l.next_token().term());
    }

    #[test]
    fn test_wrong_and() {
        let s = &mut "& ".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("&".to_string()), t.term());
    }

    #[test]
    fn test_wrong_or() {
        let s = &mut "| ".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("|".to_string()), t.term());
    }

    #[test]
    fn test_wrong_dollar() {
        let s = &mut "$ ".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("$".to_string()), t.term());
    }

    #[test]
    fn test_wrong_colon() {
        let s = &mut ": ".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error(":".to_string()), t.term());
    }

    #[test]
    fn test_wrong_plus() {
        let s = &mut "+ ".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("+".to_string()), t.term());
    }

    #[test]
    fn test_wrong_help() {
        let s = &mut "help e".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("e".to_string()), t.term());
    }

    #[test]
    fn test_wrong_help_with_nothing() {
        let s = &mut "help".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("help".to_string()), t.term());
    }

    #[test]
    fn test_no_help_string() {
        let s = &mut "help\n  \nfoo".as_bytes();
        let mut l = Lexer::create(s);
        let t = l.next_token();
        assert_eq!(Lexicon::Error("\n \n".to_string()), t.term());
    }

    #[test]
    fn test_macro_ident_at_end() {
        let s = &mut "$(foo".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(
            Lexicon::Identifier("foo".to_string()),
            l.next_token().term()
        );
    }

    #[test]
    fn test_end_half_way() {
        let s = &mut "$".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::Error("$".to_string()), l.next_token().term());
    }

    #[test]
    fn test_macro_paren_error() {
        let s = &mut "$((".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(Lexicon::Error("(".to_string()), l.next_token().term());
    }

    #[test]
    fn test_macro_ident_paren_error() {
        let s = &mut "$(foo(".as_bytes();
        let mut l = Lexer::create(s);
        assert_eq!(Lexicon::MacroOpen, l.next_token().term());
        assert_eq!(Lexicon::Error("foo(".to_string()), l.next_token().term());
    }
}