1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

//! The module tp contain all token handling logic.
#[cfg(doc)]
use crate::public_item::PublicItem;

/// A token in a rendered [`PublicItem`], used to apply syntax colouring in downstream applications.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Token {
    /// A symbol, like `=` or `::<`
    Symbol(String),
    /// A qualifier, like `pub` or `const`
    Qualifier(String),
    /// The kind of an item, like `function` or `trait`
    Kind(String),
    /// Whitespace, a single space
    Whitespace,
    /// An identifier, like variable names or parts of the path of an item
    Identifier(String),
    /// An annotation, used e.g. for Rust attributes.
    Annotation(String),
    /// The identifier self, the text can be `self` or `Self`
    Self_(String),
    /// The identifier for a function, like `fn_arg` in `comprehensive_api::functions::fn_arg`
    Function(String),
    /// A lifetime including the apostrophe `'`, like `'a`
    Lifetime(String),
    /// A keyword, like `impl`, `where`, or `dyn`
    Keyword(String),
    /// A generic parameter, like `T`
    Generic(String),
    /// A primitive type, like `usize`
    Primitive(String),
    /// A non-primitive type, like the name of a struct or a trait
    Type(String),
}

impl Token {
    /// A symbol, like `=` or `::<`
    pub(crate) fn symbol(text: impl Into<String>) -> Self {
        Self::Symbol(text.into())
    }
    /// A qualifier, like `pub` or `const`
    pub(crate) fn qualifier(text: impl Into<String>) -> Self {
        Self::Qualifier(text.into())
    }
    /// The kind of an item, like `function` or `trait`
    pub(crate) fn kind(text: impl Into<String>) -> Self {
        Self::Kind(text.into())
    }
    /// An identifier, like variable names or parts of the path of an item
    pub(crate) fn identifier(text: impl Into<String>) -> Self {
        Self::Identifier(text.into())
    }
    /// The identifier self, the text can be `self` or `Self`
    pub(crate) fn self_(text: impl Into<String>) -> Self {
        Self::Self_(text.into())
    }
    /// The identifier for a function, like `fn_arg` in `comprehensive_api::functions::fn_arg`
    pub(crate) fn function(text: impl Into<String>) -> Self {
        Self::Function(text.into())
    }
    /// A lifetime including the apostrophe `'`, like `'a`
    pub(crate) fn lifetime(text: impl Into<String>) -> Self {
        Self::Lifetime(text.into())
    }
    /// A keyword, like `impl`
    pub(crate) fn keyword(text: impl Into<String>) -> Self {
        Self::Keyword(text.into())
    }
    /// A generic, like `T`
    pub(crate) fn generic(text: impl Into<String>) -> Self {
        Self::Generic(text.into())
    }
    /// A primitive type, like `usize`
    pub(crate) fn primitive(text: impl Into<String>) -> Self {
        Self::Primitive(text.into())
    }
    /// A type, like `Iterator`
    pub(crate) fn type_(text: impl Into<String>) -> Self {
        Self::Type(text.into())
    }
    /// Give the length of the inner text of this token
    #[allow(clippy::len_without_is_empty)]
    #[must_use]
    pub fn len(&self) -> usize {
        self.text().len()
    }
    /// Get the inner text of this token
    #[must_use]
    pub fn text(&self) -> &str {
        match self {
            Self::Symbol(l)
            | Self::Qualifier(l)
            | Self::Kind(l)
            | Self::Identifier(l)
            | Self::Annotation(l)
            | Self::Self_(l)
            | Self::Function(l)
            | Self::Lifetime(l)
            | Self::Keyword(l)
            | Self::Generic(l)
            | Self::Primitive(l)
            | Self::Type(l) => l,
            Self::Whitespace => " ",
        }
    }
}

pub(crate) fn tokens_to_string(tokens: &[Token]) -> String {
    tokens.iter().map(Token::text).collect()
}