blex 0.2.2

A lightweight lexing framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use std::fmt;
use std::ops::Range;

use crate::format_tags;

#[derive(Clone, PartialEq, Debug)]

/// A representation of a token. The `body` field points to the underlying string
/// from which the characters in this token come. The `indices` field contains
/// a range representing which contiguous characters are used. The `tags` field is
/// a vector of borrowed strings with user-defined values.
pub struct Token<'a> {
    pub body: &'a str,
    pub indices: Range<usize>,
    pub tags: Vec<&'a str>,
}

impl Default for Token<'_> {
    /// A token's default value is the empty token, which borrows 0 characters
    /// from a new empty string.
    fn default() -> Self {
        empty_token()
    }
}

impl<'a> Token<'a> {
    /// Gets the content of the token as a string slice from the body. Mostly used
    /// for printing tokens.
    pub fn content(&'a self) -> &'a str {
        &self.body[self.indices.clone()]
    }

    /// Whether or not a token's tags contain a certain string slice value.
    pub fn has_tag(&self, tag: &str) -> bool {
        self.tags.contains(&tag)
    }

    /// If this token is a single character long, returns that character. Otherwise,
    /// returns None.
    pub fn single_char(&self) -> Option<char> {
        self.content().chars().next()
    }
}

/// A token with no contents and a static body.
pub fn empty_token() -> Token<'static> {
    Token {
        body: "",
        indices: 0..0,
        tags: vec![],
    }
}

impl<'a> fmt::Display for Token<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{0:?}: {1}", self.content(), format_tags(self.tags.clone()))
    }
}

/// Consolidates a single string slice and a vector of tags into one token.
/// Mostly used for debugging.
pub fn token_from_string<'a>(content: &'a str, tags: Vec<&'a str>) -> Token<'a> {
    Token {
        body: content,
        indices: 0..content.len(),
        tags,
    }
}

/// Prints a vector of tokens using their default Display method.
pub fn print_tokens(tokens: Vec<Token>) {
    for tok in tokens {
        println!("{}", tok);
    }
}

/// Chops up a string slice into a vector of owned tokens. Also appends an empty
/// token to the tail of the vector to enable certain lexing functions like
/// scanning for words.
pub fn str_to_tokens<'a>(body: &'a str) -> Vec<Token> {
    let mut tokens: Vec<Token> = vec![];
    for index in 0..body.len() {
        tokens.push(Token {
            body,
            indices: index..index + 1,
            tags: vec![&body[index..index + 1]],
        });
    }
    tokens.push(empty_token());
    tokens
}

/// Wraps up a vector of consecutive tokens into one token and applies the
/// specified tags.
pub fn wrap<'a>(tokens: Vec<Token<'a>>, tags: Vec<&'a str>) -> Token<'a> {
    if tokens.is_empty() {
        return empty_token();
    }
    Token {
        body: tokens[0].body,
        indices: tokens[0].indices.start..tokens.last().unwrap().indices.end,
        tags,
    }
}

/// Represents the possible shapes of a vector of tokens. A vector of tokens could
/// contain multiple tokens, a single token (a common special case), or none at
/// all (a rare corner case).
pub enum TokenStructure<'a> {
    Multiple,
    Single(&'a Token<'a>),
    None,
}

/// Goes with the TokenStructure enum. Specifies whether a borrowed vector of
/// tokens consists of multiple tokens, a single token, or none at all.
pub fn tokens_structure<'a>(tokens: &'a Vec<Token<'a>>) -> TokenStructure<'a> {
    if tokens.len() == 1 {
        TokenStructure::Single(&tokens[0])
    } else if tokens.len() > 1 {
        TokenStructure::Multiple
    } else {
        TokenStructure::None
    }
}