harper_core/
token_kind.rs

1use harper_brill::UPOS;
2use is_macro::Is;
3use serde::{Deserialize, Serialize};
4
5use crate::{Number, Punctuation, Quote, TokenKind::Word, WordMetadata};
6
7/// Generate wrapper code to pass a function call to the inner [`WordMetadata`],  
8/// if the token is indeed a word, while also emitting method-level documentation.
9macro_rules! delegate_to_metadata {
10    ($($method:ident),* $(,)?) => {
11        $(
12            #[doc = concat!(
13                "Delegates to [`WordMetadata::",
14                stringify!($method),
15                "`] when this token is a word.\n\n",
16                "Returns `false` if the token is not a word."
17            )]
18            pub fn $method(&self) -> bool {
19                let Word(Some(metadata)) = self else {
20                    return false;
21                };
22                metadata.$method()
23            }
24        )*
25    };
26}
27
28/// The parsed value of a [`Token`](crate::Token).
29/// Has a variety of queries available.
30/// If there is a query missing, it may be easy to implement by just calling the
31/// `delegate_to_metadata` macro.
32#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq)]
33#[serde(tag = "kind", content = "value")]
34pub enum TokenKind {
35    /// `None` if the word does not exist in the dictionary.
36    Word(Option<WordMetadata>),
37    Punctuation(Punctuation),
38    Decade,
39    Number(Number),
40    /// A sequence of " " spaces.
41    Space(usize),
42    /// A sequence of "\n" newlines
43    Newline(usize),
44    EmailAddress,
45    Url,
46    Hostname,
47    /// A special token used for things like inline code blocks that should be
48    /// ignored by all linters.
49    #[default]
50    Unlintable,
51    ParagraphBreak,
52    Regexish,
53}
54
55impl TokenKind {
56    // Word metadata delegation methods grouped by part of speech
57    delegate_to_metadata! {
58        // Nominal methods (nouns and pronouns)
59        is_nominal,
60        is_noun,
61        is_pronoun,
62        is_proper_noun,
63        is_singular_nominal,
64        is_plural_nominal,
65        is_possessive_nominal,
66        is_non_plural_nominal,
67        is_singular_noun,
68        is_plural_noun,
69        is_non_plural_noun,
70        is_countable_noun,
71        is_non_countable_noun,
72        is_mass_noun,
73        is_mass_noun_only,
74        is_non_mass_noun,
75        is_singular_pronoun,
76        is_plural_pronoun,
77        is_non_plural_pronoun,
78        is_reflexive_pronoun,
79        is_first_person_singular_pronoun,
80        is_first_person_plural_pronoun,
81        is_second_person_pronoun,
82        is_third_person_pronoun,
83        is_third_person_singular_pronoun,
84        is_third_person_plural_pronoun,
85        is_object_pronoun,
86        is_possessive_noun,
87        is_possessive_pronoun,
88
89        // Verb methods
90        is_verb,
91        is_auxiliary_verb,
92        is_linking_verb,
93        is_verb_lemma,
94        is_verb_past_form,
95        is_verb_progressive_form,
96        is_verb_third_person_singular_present_form,
97
98        // Adjective methods
99        is_adjective,
100        is_comparative_adjective,
101        is_superlative_adjective,
102
103        // Adverb methods
104        is_adverb,
105
106        // Determiner methods
107        is_determiner,
108        is_demonstrative_determiner,
109        is_possessive_determiner,
110        is_quantifier,
111        is_non_quantifier_determiner,
112
113        // Conjunction methods
114        is_conjunction,
115
116        // Generic word methods
117        is_swear,
118        is_likely_homograph,
119
120        // Orthography methods
121        is_lowercase,
122        is_titlecase,
123        is_allcaps,
124        is_lower_camel,
125        is_upper_camel
126    }
127
128    // Word metadata delegation methods not generated by macro
129    pub fn is_preposition(&self) -> bool {
130        let Word(Some(metadata)) = self else {
131            return false;
132        };
133        metadata.preposition
134    }
135
136    pub fn is_common_word(&self) -> bool {
137        let Word(Some(metadata)) = self else {
138            return true;
139        };
140        metadata.common
141    }
142
143    /// Checks whether a word token is out-of-vocabulary (not found in the dictionary).
144    ///
145    /// Returns `true` if the token is a word that was not found in the dictionary,
146    /// `false` if the token is a word found in the dictionary or is not a word token.
147    pub fn is_oov(&self) -> bool {
148        matches!(self, TokenKind::Word(None))
149    }
150
151    // Punctuation and symbol is-methods
152
153    pub fn is_open_square(&self) -> bool {
154        matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare))
155    }
156
157    pub fn is_close_square(&self) -> bool {
158        matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare))
159    }
160
161    pub fn is_pipe(&self) -> bool {
162        matches!(self, TokenKind::Punctuation(Punctuation::Pipe))
163    }
164
165    pub fn is_currency(&self) -> bool {
166        matches!(self, TokenKind::Punctuation(Punctuation::Currency(..)))
167    }
168
169    pub fn is_ellipsis(&self) -> bool {
170        matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis))
171    }
172
173    pub fn is_hyphen(&self) -> bool {
174        matches!(self, TokenKind::Punctuation(Punctuation::Hyphen))
175    }
176
177    pub fn is_quote(&self) -> bool {
178        matches!(self, TokenKind::Punctuation(Punctuation::Quote(_)))
179    }
180
181    pub fn is_apostrophe(&self) -> bool {
182        matches!(self, TokenKind::Punctuation(Punctuation::Apostrophe))
183    }
184
185    pub fn is_period(&self) -> bool {
186        matches!(self, TokenKind::Punctuation(Punctuation::Period))
187    }
188
189    pub fn is_at(&self) -> bool {
190        matches!(self, TokenKind::Punctuation(Punctuation::At))
191    }
192
193    pub fn is_comma(&self) -> bool {
194        matches!(self, TokenKind::Punctuation(Punctuation::Comma))
195    }
196
197    pub fn is_semicolon(&self) -> bool {
198        matches!(self, TokenKind::Punctuation(Punctuation::Semicolon))
199    }
200
201    // Miscellaneous is-methods
202
203    /// Checks whether a token is word-like--meaning it is more complex than punctuation and can
204    /// hold semantic meaning in the way a word does.
205    pub fn is_word_like(&self) -> bool {
206        matches!(
207            self,
208            TokenKind::Word(..)
209                | TokenKind::EmailAddress
210                | TokenKind::Hostname
211                | TokenKind::Decade
212                | TokenKind::Number(..)
213        )
214    }
215
216    pub(crate) fn is_chunk_terminator(&self) -> bool {
217        if self.is_sentence_terminator() {
218            return true;
219        }
220
221        match self {
222            TokenKind::Punctuation(punct) => {
223                matches!(
224                    punct,
225                    Punctuation::Comma | Punctuation::Quote { .. } | Punctuation::Colon
226                )
227            }
228            _ => false,
229        }
230    }
231
232    pub(crate) fn is_sentence_terminator(&self) -> bool {
233        match self {
234            TokenKind::Punctuation(punct) => [
235                Punctuation::Period,
236                Punctuation::Bang,
237                Punctuation::Question,
238            ]
239            .contains(punct),
240            TokenKind::ParagraphBreak => true,
241            _ => false,
242        }
243    }
244
245    /// Used by `crate::parsers::CollapseIdentifiers`
246    /// TODO: Separate this into two functions and add OR functionality to
247    /// pattern matching
248    pub fn is_case_separator(&self) -> bool {
249        matches!(self, TokenKind::Punctuation(Punctuation::Underscore))
250            || matches!(self, TokenKind::Punctuation(Punctuation::Hyphen))
251    }
252
253    /// Checks whether the token is whitespace.
254    pub fn is_whitespace(&self) -> bool {
255        matches!(self, TokenKind::Space(_) | TokenKind::Newline(_))
256    }
257
258    pub fn is_upos(&self, upos: UPOS) -> bool {
259        let Some(Some(meta)) = self.as_word() else {
260            return false;
261        };
262
263        meta.pos_tag == Some(upos)
264    }
265
266    // Miscellaneous non-is methods
267
268    /// Checks that `self` is the same enum variant as `other`, regardless of
269    /// whether the inner metadata is also equal.
270    pub fn matches_variant_of(&self, other: &Self) -> bool {
271        self.with_default_data() == other.with_default_data()
272    }
273
274    /// Produces a copy of `self` with any inner data replaced with its default
275    /// value. Useful for making comparisons on just the variant of the
276    /// enum.
277    pub fn with_default_data(&self) -> Self {
278        match self {
279            TokenKind::Word(_) => TokenKind::Word(Default::default()),
280            TokenKind::Punctuation(_) => TokenKind::Punctuation(Default::default()),
281            TokenKind::Number(..) => TokenKind::Number(Default::default()),
282            TokenKind::Space(_) => TokenKind::Space(Default::default()),
283            TokenKind::Newline(_) => TokenKind::Newline(Default::default()),
284            _ => self.clone(),
285        }
286    }
287
288    /// Construct a [`TokenKind::Word`] with no metadata.
289    pub fn blank_word() -> Self {
290        Self::Word(None)
291    }
292
293    // Punctuation and symbol non-is methods
294
295    pub fn as_mut_quote(&mut self) -> Option<&mut Quote> {
296        self.as_mut_punctuation()?.as_mut_quote()
297    }
298
299    pub fn as_quote(&self) -> Option<&Quote> {
300        self.as_punctuation()?.as_quote()
301    }
302}
303
304#[cfg(test)]
305mod tests {
306    use crate::Document;
307
308    #[test]
309    fn car_is_singular_noun() {
310        let doc = Document::new_plain_english_curated("car");
311        let tk = &doc.tokens().next().unwrap().kind;
312        assert!(tk.is_singular_noun());
313    }
314
315    #[test]
316    fn traffic_is_mass_noun_only() {
317        let doc = Document::new_plain_english_curated("traffic");
318        let tk = &doc.tokens().next().unwrap().kind;
319        assert!(tk.is_mass_noun_only());
320    }
321
322    #[test]
323    fn equipment_is_mass_noun() {
324        let doc = Document::new_plain_english_curated("equipment");
325        let tk = &doc.tokens().next().unwrap().kind;
326        assert!(tk.is_mass_noun());
327    }
328
329    #[test]
330    fn equipment_is_non_countable_noun() {
331        let doc = Document::new_plain_english_curated("equipment");
332        let tk = &doc.tokens().next().unwrap().kind;
333        assert!(tk.is_non_countable_noun());
334    }
335
336    #[test]
337    fn equipment_isnt_countable_noun() {
338        let doc = Document::new_plain_english_curated("equipment");
339        let tk = &doc.tokens().next().unwrap().kind;
340        assert!(!tk.is_countable_noun());
341    }
342}
harper_core/token_kind.rs

harper_core/
token_kind.rs
