erl_tokenize 0.10.0

Erlang source code tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

use std::fmt;

use crate::tokens::{CommentToken, WhitespaceToken};
use crate::{Position, PositionRange};

/// Hidden token.
///
/// "Hidden" means it has no lexical meanings.
#[allow(missing_docs)]
#[derive(Debug, Clone)]
pub enum HiddenToken {
    Comment(CommentToken),
    Whitespace(WhitespaceToken),
}
impl HiddenToken {
    /// Returns the original textual representation of this token.
    pub fn text(&self) -> &str {
        match *self {
            HiddenToken::Comment(ref t) => t.text(),
            HiddenToken::Whitespace(ref t) => t.text(),
        }
    }

    /// Tries to return the reference to the inner `CommentToken`.
    pub fn as_comment_token(&self) -> Option<&CommentToken> {
        if let HiddenToken::Comment(ref t) = *self {
            Some(t)
        } else {
            None
        }
    }

    /// Tries to return the reference to the inner `WhitespaceToken`.
    pub fn as_whitespace_token(&self) -> Option<&WhitespaceToken> {
        if let HiddenToken::Whitespace(ref t) = *self {
            Some(t)
        } else {
            None
        }
    }

    /// Tries to return the inner `CommentToken`.
    pub fn into_comment_token(self) -> Result<CommentToken, Self> {
        if let HiddenToken::Comment(t) = self {
            Ok(t)
        } else {
            Err(self)
        }
    }

    /// Tries to return the inner `WhitespaceToken`.
    pub fn into_whitespace_token(self) -> Result<WhitespaceToken, Self> {
        if let HiddenToken::Whitespace(t) = self {
            Ok(t)
        } else {
            Err(self)
        }
    }
}
impl From<CommentToken> for HiddenToken {
    fn from(f: CommentToken) -> Self {
        HiddenToken::Comment(f)
    }
}
impl From<WhitespaceToken> for HiddenToken {
    fn from(f: WhitespaceToken) -> Self {
        HiddenToken::Whitespace(f)
    }
}
impl PositionRange for HiddenToken {
    fn start_position(&self) -> Position {
        match *self {
            HiddenToken::Comment(ref t) => t.start_position(),
            HiddenToken::Whitespace(ref t) => t.start_position(),
        }
    }
    fn end_position(&self) -> Position {
        match *self {
            HiddenToken::Comment(ref t) => t.end_position(),
            HiddenToken::Whitespace(ref t) => t.end_position(),
        }
    }
}
impl fmt::Display for HiddenToken {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.text().fmt(f)
    }
}