luthor 0.2.0

A collection of lexers for various languages/formats, and the tools required to build them.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

//! Token-related types.

/// The primary means of classifying a format or language's lexemes.
#[derive(PartialEq, Debug, Clone)]
pub enum Category {
    Whitespace,
    Identifier,
    Keyword,
    Brace,
    Bracket,
    Parenthesis,
    Operator,
    Integer,
    Float,
    String,
    Boolean,
    Text,
    Comment,
    Function,
    Method,
    Call,
    Literal,
    Key,
}

/// A lexeme and category pairing. Tokens are the final product of a lexer; 
/// their lexemes should join to produce the original data passed to the lexer.
#[derive(PartialEq, Debug, Clone)]
pub struct Token {
    pub lexeme: String,
    pub category: Category,
}