harper_core/
token_kind.rs

1use harper_brill::UPOS;
2use is_macro::Is;
3use serde::{Deserialize, Serialize};
4
5use crate::{
6    DictWordMetadata, Number, Punctuation, Quote, TokenKind::Word, dict_word_metadata::Person,
7};
8
9/// Generate wrapper code to pass a function call to the inner [`DictWordMetadata`],  
10/// if the token is indeed a word, while also emitting method-level documentation.
11macro_rules! delegate_to_metadata {
12    ($($method:ident),* $(,)?) => {
13        $(
14            #[doc = concat!(
15                "Delegates to [`DictWordMetadata::",
16                stringify!($method),
17                "`] when this token is a word.\n\n",
18                "Returns `false` if the token is not a word."
19            )]
20            pub fn $method(&self) -> bool {
21                let Word(Some(metadata)) = self else {
22                    return false;
23                };
24                metadata.$method()
25            }
26        )*
27    };
28}
29
30/// The parsed value of a [`Token`](crate::Token).
31/// Has a variety of queries available.
32/// If there is a query missing, it may be easy to implement by just calling the
33/// `delegate_to_metadata` macro.
34#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq)]
35#[serde(tag = "kind", content = "value")]
36pub enum TokenKind {
37    /// `None` if the word does not exist in the dictionary.
38    Word(Option<DictWordMetadata>),
39    Punctuation(Punctuation),
40    Decade,
41    Number(Number),
42    /// A sequence of " " spaces.
43    Space(usize),
44    /// A sequence of "\n" newlines
45    Newline(usize),
46    EmailAddress,
47    Url,
48    Hostname,
49    /// A special token used for things like inline code blocks that should be
50    /// ignored by all linters.
51    #[default]
52    Unlintable,
53    ParagraphBreak,
54    Regexish,
55    HeadingStart,
56}
57
58impl TokenKind {
59    // DictWord metadata delegation methods grouped by part of speech
60    delegate_to_metadata! {
61        // Nominal methods (nouns and pronouns)
62        is_nominal,
63        is_noun,
64        is_pronoun,
65        is_proper_noun,
66        is_singular_nominal,
67        is_plural_nominal,
68        is_possessive_nominal,
69        is_non_plural_nominal,
70        is_singular_noun,
71        is_singular_noun_only,
72        is_plural_noun,
73        is_plural_noun_only,
74        is_non_plural_noun,
75        is_non_possessive_noun,
76        is_countable_noun,
77        is_non_countable_noun,
78        is_mass_noun,
79        is_mass_noun_only,
80        is_non_mass_noun,
81        is_singular_pronoun,
82        is_plural_pronoun,
83        is_non_plural_pronoun,
84        is_reflexive_pronoun,
85        is_personal_pronoun,
86        is_first_person_singular_pronoun,
87        is_first_person_plural_pronoun,
88        is_second_person_pronoun,
89        is_third_person_pronoun,
90        is_third_person_singular_pronoun,
91        is_third_person_plural_pronoun,
92        is_subject_pronoun,
93        is_object_pronoun,
94        is_possessive_noun,
95        // Note: possessive pronouns are: mine, ours, yours, his, hers, its, theirs
96        is_possessive_pronoun,
97
98        // Verb methods
99        is_verb,
100        is_auxiliary_verb,
101        is_linking_verb,
102        is_verb_lemma,
103        is_verb_past_form,
104        is_verb_regular_past_form,
105        is_verb_simple_past_form,
106        is_verb_past_participle_form,
107        is_verb_simple_past_only,
108        is_verb_past_participle_only,
109        is_verb_progressive_form,
110        is_verb_third_person_singular_present_form,
111
112        // Adjective methods
113        is_adjective,
114        is_comparative_adjective,
115        is_superlative_adjective,
116        is_positive_adjective,
117
118        // Adverb methods
119        is_adverb,
120        is_manner_adverb,
121        is_frequency_adverb,
122        is_degree_adverb,
123
124        // Determiner methods
125        is_determiner,
126        is_demonstrative_determiner,
127        is_possessive_determiner,
128        is_quantifier,
129        is_non_quantifier_determiner,
130        is_non_demonstrative_determiner,
131
132        // Conjunction methods
133        is_conjunction,
134
135        // Generic word methods
136        is_swear,
137        is_abbreviation,
138        is_likely_homograph,
139
140        // Orthography methods
141        is_lowercase,
142        is_titlecase,
143        is_allcaps,
144        is_lower_camel,
145        is_upper_camel,
146        is_apostrophized,
147
148        is_roman_numerals
149    }
150
151    pub fn get_pronoun_person(&self) -> Option<Person> {
152        let Word(Some(metadata)) = self else {
153            return None;
154        };
155        metadata.get_person()
156    }
157
158    // DictWord metadata delegation methods not generated by macro
159    pub fn is_preposition(&self) -> bool {
160        let Word(Some(metadata)) = self else {
161            return false;
162        };
163        metadata.preposition
164    }
165
166    // Generic word is-methods
167
168    pub fn is_common_word(&self) -> bool {
169        let Word(Some(metadata)) = self else {
170            return true;
171        };
172        metadata.common
173    }
174
175    /// Checks whether the token is a member of a nominal phrase.
176    pub fn is_np_member(&self) -> bool {
177        let Word(Some(metadata)) = self else {
178            return false;
179        };
180        metadata.np_member.unwrap_or(false)
181    }
182
183    /// Checks whether a word token is out-of-vocabulary (not found in the dictionary).
184    ///
185    /// Returns `true` if the token is a word that was not found in the dictionary,
186    /// `false` if the token is a word found in the dictionary or is not a word token.
187    pub fn is_oov(&self) -> bool {
188        matches!(self, TokenKind::Word(None))
189    }
190
191    // Number is-methods
192
193    pub fn is_cardinal_number(&self) -> bool {
194        matches!(self, TokenKind::Number(Number { suffix: None, .. }))
195    }
196
197    pub fn is_ordinal_number(&self) -> bool {
198        matches!(
199            self,
200            TokenKind::Number(Number {
201                suffix: Some(_),
202                ..
203            })
204        )
205    }
206
207    // Punctuation and symbol is-methods
208
209    pub fn is_open_square(&self) -> bool {
210        matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare))
211    }
212
213    pub fn is_close_square(&self) -> bool {
214        matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare))
215    }
216
217    pub fn is_less_than(&self) -> bool {
218        matches!(self, TokenKind::Punctuation(Punctuation::LessThan))
219    }
220
221    pub fn is_greater_than(&self) -> bool {
222        matches!(self, TokenKind::Punctuation(Punctuation::GreaterThan))
223    }
224
225    pub fn is_open_round(&self) -> bool {
226        matches!(self, TokenKind::Punctuation(Punctuation::OpenRound))
227    }
228
229    pub fn is_close_round(&self) -> bool {
230        matches!(self, TokenKind::Punctuation(Punctuation::CloseRound))
231    }
232
233    pub fn is_pipe(&self) -> bool {
234        matches!(self, TokenKind::Punctuation(Punctuation::Pipe))
235    }
236
237    pub fn is_currency(&self) -> bool {
238        matches!(self, TokenKind::Punctuation(Punctuation::Currency(..)))
239    }
240
241    pub fn is_ellipsis(&self) -> bool {
242        matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis))
243    }
244
245    // AKA 'minus'
246    pub fn is_hyphen(&self) -> bool {
247        matches!(self, TokenKind::Punctuation(Punctuation::Hyphen))
248    }
249
250    pub fn is_plus(&self) -> bool {
251        matches!(self, TokenKind::Punctuation(Punctuation::Plus))
252    }
253
254    pub fn is_quote(&self) -> bool {
255        matches!(self, TokenKind::Punctuation(Punctuation::Quote(_)))
256    }
257
258    pub fn is_apostrophe(&self) -> bool {
259        matches!(self, TokenKind::Punctuation(Punctuation::Apostrophe))
260    }
261
262    pub fn is_period(&self) -> bool {
263        matches!(self, TokenKind::Punctuation(Punctuation::Period))
264    }
265
266    pub fn is_at(&self) -> bool {
267        matches!(self, TokenKind::Punctuation(Punctuation::At))
268    }
269
270    pub fn is_comma(&self) -> bool {
271        matches!(self, TokenKind::Punctuation(Punctuation::Comma))
272    }
273
274    pub fn is_semicolon(&self) -> bool {
275        matches!(self, TokenKind::Punctuation(Punctuation::Semicolon))
276    }
277
278    pub fn is_acute(&self) -> bool {
279        matches!(self, TokenKind::Punctuation(Punctuation::Acute))
280    }
281
282    pub fn is_ampersand(&self) -> bool {
283        matches!(self, TokenKind::Punctuation(Punctuation::Ampersand))
284    }
285
286    pub fn is_backslash(&self) -> bool {
287        matches!(self, TokenKind::Punctuation(Punctuation::Backslash))
288    }
289
290    pub fn is_slash(&self) -> bool {
291        matches!(self, TokenKind::Punctuation(Punctuation::ForwardSlash))
292    }
293
294    pub fn is_percent(&self) -> bool {
295        matches!(self, TokenKind::Punctuation(Punctuation::Percent))
296    }
297
298    pub fn is_degree(&self) -> bool {
299        matches!(self, TokenKind::Punctuation(Punctuation::Degree))
300    }
301
302    pub fn is_open_single(&self) -> bool {
303        matches!(self, TokenKind::Punctuation(Punctuation::OpenSingle))
304    }
305
306    pub fn is_single_prime(&self) -> bool {
307        matches!(self, TokenKind::Punctuation(Punctuation::SinglePrime))
308    }
309
310    pub fn is_double_prime(&self) -> bool {
311        matches!(self, TokenKind::Punctuation(Punctuation::DoublePrime))
312    }
313
314    pub fn is_backtick(&self) -> bool {
315        matches!(self, TokenKind::Punctuation(Punctuation::Backtick))
316    }
317
318    // Miscellaneous is-methods
319
320    /// Checks whether a token is word-like--meaning it is more complex than punctuation and can
321    /// hold semantic meaning in the way a word does.
322    pub fn is_word_like(&self) -> bool {
323        matches!(
324            self,
325            TokenKind::Word(..)
326                | TokenKind::EmailAddress
327                | TokenKind::Hostname
328                | TokenKind::Decade
329                | TokenKind::Number(..)
330        )
331    }
332
333    pub(crate) fn is_chunk_terminator(&self) -> bool {
334        if self.is_sentence_terminator() {
335            return true;
336        }
337
338        match self {
339            TokenKind::Punctuation(punct) => {
340                matches!(
341                    punct,
342                    Punctuation::Comma
343                        | Punctuation::Semicolon
344                        | Punctuation::Quote { .. }
345                        | Punctuation::Colon
346                )
347            }
348            _ => false,
349        }
350    }
351
352    pub fn is_sentence_terminator(&self) -> bool {
353        match self {
354            TokenKind::Punctuation(punct) => [
355                Punctuation::Period,
356                Punctuation::Bang,
357                Punctuation::Question,
358            ]
359            .contains(punct),
360            TokenKind::ParagraphBreak => true,
361            _ => false,
362        }
363    }
364
365    /// Used by `crate::parsers::CollapseIdentifiers`
366    /// TODO: Separate this into two functions and add OR functionality to
367    /// pattern matching
368    pub fn is_case_separator(&self) -> bool {
369        matches!(self, TokenKind::Punctuation(Punctuation::Underscore))
370            || matches!(self, TokenKind::Punctuation(Punctuation::Hyphen))
371    }
372
373    /// Checks whether the token is whitespace.
374    pub fn is_whitespace(&self) -> bool {
375        matches!(self, TokenKind::Space(_) | TokenKind::Newline(_))
376    }
377
378    pub fn is_upos(&self, upos: UPOS) -> bool {
379        let Some(Some(meta)) = self.as_word() else {
380            return false;
381        };
382
383        meta.pos_tag == Some(upos)
384    }
385
386    // Miscellaneous non-is methods
387
388    /// Checks that `self` is the same enum variant as `other`, regardless of
389    /// whether the inner metadata is also equal.
390    pub fn matches_variant_of(&self, other: &Self) -> bool {
391        self.with_default_data() == other.with_default_data()
392    }
393
394    /// Produces a copy of `self` with any inner data replaced with its default
395    /// value. Useful for making comparisons on just the variant of the
396    /// enum.
397    pub fn with_default_data(&self) -> Self {
398        match self {
399            TokenKind::Word(_) => TokenKind::Word(Default::default()),
400            TokenKind::Punctuation(_) => TokenKind::Punctuation(Default::default()),
401            TokenKind::Number(..) => TokenKind::Number(Default::default()),
402            TokenKind::Space(_) => TokenKind::Space(Default::default()),
403            TokenKind::Newline(_) => TokenKind::Newline(Default::default()),
404            _ => self.clone(),
405        }
406    }
407
408    /// Construct a [`TokenKind::Word`] with no metadata.
409    pub fn blank_word() -> Self {
410        Self::Word(None)
411    }
412
413    // Punctuation and symbol non-is methods
414
415    pub fn as_mut_quote(&mut self) -> Option<&mut Quote> {
416        self.as_mut_punctuation()?.as_mut_quote()
417    }
418
419    pub fn as_quote(&self) -> Option<&Quote> {
420        self.as_punctuation()?.as_quote()
421    }
422}
423
424#[cfg(test)]
425mod tests {
426    use crate::Document;
427
428    #[test]
429    fn car_is_singular_noun() {
430        let doc = Document::new_plain_english_curated("car");
431        let tk = &doc.tokens().next().unwrap().kind;
432        assert!(tk.is_singular_noun());
433    }
434
435    #[test]
436    fn traffic_is_mass_noun_only() {
437        let doc = Document::new_plain_english_curated("traffic");
438        let tk = &doc.tokens().next().unwrap().kind;
439        assert!(tk.is_mass_noun_only());
440    }
441
442    #[test]
443    fn equipment_is_mass_noun() {
444        let doc = Document::new_plain_english_curated("equipment");
445        let tk = &doc.tokens().next().unwrap().kind;
446        assert!(tk.is_mass_noun());
447    }
448
449    #[test]
450    fn equipment_is_non_countable_noun() {
451        let doc = Document::new_plain_english_curated("equipment");
452        let tk = &doc.tokens().next().unwrap().kind;
453        assert!(tk.is_non_countable_noun());
454    }
455
456    #[test]
457    fn equipment_isnt_countable_noun() {
458        let doc = Document::new_plain_english_curated("equipment");
459        let tk = &doc.tokens().next().unwrap().kind;
460        assert!(!tk.is_countable_noun());
461    }
462
463    #[test]
464    fn ate_is_simple_past_only() {
465        let doc = Document::new_plain_english_curated("ate");
466        let tk = &doc.tokens().next().unwrap().kind;
467        assert!(tk.is_verb_simple_past_only());
468        assert!(!tk.is_verb_past_participle_only());
469    }
470
471    #[test]
472    fn eaten_is_past_participle_only() {
473        let doc = Document::new_plain_english_curated("eaten");
474        let tk = &doc.tokens().next().unwrap().kind;
475        assert!(tk.is_verb_past_participle_only());
476        assert!(!tk.is_verb_simple_past_only());
477    }
478
479    #[test]
480    fn thought_is_regular_past_form() {
481        let doc = Document::new_plain_english_curated("thought");
482        let tk = &doc.tokens().next().unwrap().kind;
483        assert!(tk.is_verb_regular_past_form());
484    }
485
486    #[test]
487    fn oov_word_is_oov() {
488        let doc = Document::new_plain_english_curated("nonexistentword");
489        let tk = &doc.tokens().next().unwrap().kind;
490        assert!(tk.is_oov());
491    }
492
493    #[test]
494    fn known_word_is_not_oov() {
495        let doc = Document::new_plain_english_curated("car");
496        let tk = &doc.tokens().next().unwrap().kind;
497        assert!(!tk.is_oov());
498    }
499
500    #[test]
501    fn non_word_tokens_are_not_oov() {
502        let doc = Document::new_plain_english_curated("Hello, world!");
503        let tokens: Vec<_> = doc.tokens().collect();
504
505        // Comma should not be OOV
506        assert!(!tokens[1].kind.is_oov());
507        // Exclamation mark should not be OOV
508        assert!(!tokens[3].kind.is_oov());
509    }
510}
harper_core/token_kind.rs

harper_core/
token_kind.rs
