harper_core/
token_kind.rs

1use harper_brill::UPOS;
2use is_macro::Is;
3use serde::{Deserialize, Serialize};
4
5use crate::{
6    DictWordMetadata, Number, Punctuation, Quote, TokenKind::Word, dict_word_metadata::Person,
7};
8
9/// Generate wrapper code to pass a function call to the inner [`DictWordMetadata`],  
10/// if the token is indeed a word, while also emitting method-level documentation.
11macro_rules! delegate_to_metadata {
12    ($($method:ident),* $(,)?) => {
13        $(
14            #[doc = concat!(
15                "Delegates to [`DictWordMetadata::",
16                stringify!($method),
17                "`] when this token is a word.\n\n",
18                "Returns `false` if the token is not a word."
19            )]
20            pub fn $method(&self) -> bool {
21                let Word(Some(metadata)) = self else {
22                    return false;
23                };
24                metadata.$method()
25            }
26        )*
27    };
28}
29
30/// The parsed value of a [`Token`](crate::Token).
31/// Has a variety of queries available.
32/// If there is a query missing, it may be easy to implement by just calling the
33/// `delegate_to_metadata` macro.
34#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq)]
35#[serde(tag = "kind", content = "value")]
36pub enum TokenKind {
37    /// `None` if the word does not exist in the dictionary.
38    Word(Option<DictWordMetadata>),
39    Punctuation(Punctuation),
40    Decade,
41    Number(Number),
42    /// A sequence of " " spaces.
43    Space(usize),
44    /// A sequence of "\n" newlines
45    Newline(usize),
46    EmailAddress,
47    Url,
48    Hostname,
49    /// A special token used for things like inline code blocks that should be
50    /// ignored by all linters.
51    #[default]
52    Unlintable,
53    ParagraphBreak,
54    Regexish,
55    HeadingStart,
56}
57
58impl TokenKind {
59    // DictWord metadata delegation methods grouped by part of speech
60    delegate_to_metadata! {
61        // Nominal methods (nouns and pronouns)
62        is_nominal,
63        is_noun,
64        is_pronoun,
65        is_proper_noun,
66        is_singular_nominal,
67        is_plural_nominal,
68        is_possessive_nominal,
69        is_non_plural_nominal,
70        is_singular_noun,
71        is_plural_noun,
72        is_non_plural_noun,
73        is_non_possessive_noun,
74        is_countable_noun,
75        is_non_countable_noun,
76        is_mass_noun,
77        is_mass_noun_only,
78        is_non_mass_noun,
79        is_singular_pronoun,
80        is_plural_pronoun,
81        is_non_plural_pronoun,
82        is_reflexive_pronoun,
83        is_personal_pronoun,
84        is_first_person_singular_pronoun,
85        is_first_person_plural_pronoun,
86        is_second_person_pronoun,
87        is_third_person_pronoun,
88        is_third_person_singular_pronoun,
89        is_third_person_plural_pronoun,
90        is_subject_pronoun,
91        is_object_pronoun,
92        is_possessive_noun,
93        // Note: possessive pronouns are: mine, ours, yours, his, hers, its, theirs
94        is_possessive_pronoun,
95
96        // Verb methods
97        is_verb,
98        is_auxiliary_verb,
99        is_linking_verb,
100        is_verb_lemma,
101        is_verb_past_form,
102        is_verb_regular_past_form,
103        is_verb_simple_past_form,
104        is_verb_past_participle_form,
105        is_verb_simple_past_only,
106        is_verb_past_participle_only,
107        is_verb_progressive_form,
108        is_verb_third_person_singular_present_form,
109
110        // Adjective methods
111        is_adjective,
112        is_comparative_adjective,
113        is_superlative_adjective,
114        is_positive_adjective,
115
116        // Adverb methods
117        is_adverb,
118        is_manner_adverb,
119        is_frequency_adverb,
120        is_degree_adverb,
121
122        // Determiner methods
123        is_determiner,
124        is_demonstrative_determiner,
125        is_possessive_determiner,
126        is_quantifier,
127        is_non_quantifier_determiner,
128        is_non_demonstrative_determiner,
129
130        // Conjunction methods
131        is_conjunction,
132
133        // Generic word methods
134        is_swear,
135        is_likely_homograph,
136
137        // Orthography methods
138        is_lowercase,
139        is_titlecase,
140        is_allcaps,
141        is_lower_camel,
142        is_upper_camel,
143        is_apostrophized,
144
145        is_roman_numerals
146    }
147
148    pub fn get_pronoun_person(&self) -> Option<Person> {
149        let Word(Some(metadata)) = self else {
150            return None;
151        };
152        metadata.get_person()
153    }
154
155    // DictWord metadata delegation methods not generated by macro
156    pub fn is_preposition(&self) -> bool {
157        let Word(Some(metadata)) = self else {
158            return false;
159        };
160        metadata.preposition
161    }
162
163    // Generic word is-methods
164
165    pub fn is_common_word(&self) -> bool {
166        let Word(Some(metadata)) = self else {
167            return true;
168        };
169        metadata.common
170    }
171
172    /// Checks whether the token is a member of a nominal phrase.
173    pub fn is_np_member(&self) -> bool {
174        let Word(Some(metadata)) = self else {
175            return false;
176        };
177        metadata.np_member.unwrap_or(false)
178    }
179
180    /// Checks whether a word token is out-of-vocabulary (not found in the dictionary).
181    ///
182    /// Returns `true` if the token is a word that was not found in the dictionary,
183    /// `false` if the token is a word found in the dictionary or is not a word token.
184    pub fn is_oov(&self) -> bool {
185        matches!(self, TokenKind::Word(None))
186    }
187
188    // Number is-methods
189
190    pub fn is_cardinal_number(&self) -> bool {
191        matches!(self, TokenKind::Number(Number { suffix: None, .. }))
192    }
193
194    pub fn is_ordinal_number(&self) -> bool {
195        matches!(
196            self,
197            TokenKind::Number(Number {
198                suffix: Some(_),
199                ..
200            })
201        )
202    }
203
204    // Punctuation and symbol is-methods
205
206    pub fn is_open_square(&self) -> bool {
207        matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare))
208    }
209
210    pub fn is_close_square(&self) -> bool {
211        matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare))
212    }
213
214    pub fn is_less_than(&self) -> bool {
215        matches!(self, TokenKind::Punctuation(Punctuation::LessThan))
216    }
217
218    pub fn is_greater_than(&self) -> bool {
219        matches!(self, TokenKind::Punctuation(Punctuation::GreaterThan))
220    }
221
222    pub fn is_open_round(&self) -> bool {
223        matches!(self, TokenKind::Punctuation(Punctuation::OpenRound))
224    }
225
226    pub fn is_close_round(&self) -> bool {
227        matches!(self, TokenKind::Punctuation(Punctuation::CloseRound))
228    }
229
230    pub fn is_pipe(&self) -> bool {
231        matches!(self, TokenKind::Punctuation(Punctuation::Pipe))
232    }
233
234    pub fn is_currency(&self) -> bool {
235        matches!(self, TokenKind::Punctuation(Punctuation::Currency(..)))
236    }
237
238    pub fn is_ellipsis(&self) -> bool {
239        matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis))
240    }
241
242    // AKA 'minus'
243    pub fn is_hyphen(&self) -> bool {
244        matches!(self, TokenKind::Punctuation(Punctuation::Hyphen))
245    }
246
247    pub fn is_plus(&self) -> bool {
248        matches!(self, TokenKind::Punctuation(Punctuation::Plus))
249    }
250
251    pub fn is_quote(&self) -> bool {
252        matches!(self, TokenKind::Punctuation(Punctuation::Quote(_)))
253    }
254
255    pub fn is_apostrophe(&self) -> bool {
256        matches!(self, TokenKind::Punctuation(Punctuation::Apostrophe))
257    }
258
259    pub fn is_period(&self) -> bool {
260        matches!(self, TokenKind::Punctuation(Punctuation::Period))
261    }
262
263    pub fn is_at(&self) -> bool {
264        matches!(self, TokenKind::Punctuation(Punctuation::At))
265    }
266
267    pub fn is_comma(&self) -> bool {
268        matches!(self, TokenKind::Punctuation(Punctuation::Comma))
269    }
270
271    pub fn is_semicolon(&self) -> bool {
272        matches!(self, TokenKind::Punctuation(Punctuation::Semicolon))
273    }
274
275    pub fn is_acute(&self) -> bool {
276        matches!(self, TokenKind::Punctuation(Punctuation::Acute))
277    }
278
279    pub fn is_ampersand(&self) -> bool {
280        matches!(self, TokenKind::Punctuation(Punctuation::Ampersand))
281    }
282
283    pub fn is_backslash(&self) -> bool {
284        matches!(self, TokenKind::Punctuation(Punctuation::Backslash))
285    }
286
287    pub fn is_slash(&self) -> bool {
288        matches!(self, TokenKind::Punctuation(Punctuation::ForwardSlash))
289    }
290
291    pub fn is_percent(&self) -> bool {
292        matches!(self, TokenKind::Punctuation(Punctuation::Percent))
293    }
294
295    pub fn is_backtick(&self) -> bool {
296        matches!(self, TokenKind::Punctuation(Punctuation::Backtick))
297    }
298
299    // Miscellaneous is-methods
300
301    /// Checks whether a token is word-like--meaning it is more complex than punctuation and can
302    /// hold semantic meaning in the way a word does.
303    pub fn is_word_like(&self) -> bool {
304        matches!(
305            self,
306            TokenKind::Word(..)
307                | TokenKind::EmailAddress
308                | TokenKind::Hostname
309                | TokenKind::Decade
310                | TokenKind::Number(..)
311        )
312    }
313
314    pub(crate) fn is_chunk_terminator(&self) -> bool {
315        if self.is_sentence_terminator() {
316            return true;
317        }
318
319        match self {
320            TokenKind::Punctuation(punct) => {
321                matches!(
322                    punct,
323                    Punctuation::Comma
324                        | Punctuation::Semicolon
325                        | Punctuation::Quote { .. }
326                        | Punctuation::Colon
327                )
328            }
329            _ => false,
330        }
331    }
332
333    pub fn is_sentence_terminator(&self) -> bool {
334        match self {
335            TokenKind::Punctuation(punct) => [
336                Punctuation::Period,
337                Punctuation::Bang,
338                Punctuation::Question,
339            ]
340            .contains(punct),
341            TokenKind::ParagraphBreak => true,
342            _ => false,
343        }
344    }
345
346    /// Used by `crate::parsers::CollapseIdentifiers`
347    /// TODO: Separate this into two functions and add OR functionality to
348    /// pattern matching
349    pub fn is_case_separator(&self) -> bool {
350        matches!(self, TokenKind::Punctuation(Punctuation::Underscore))
351            || matches!(self, TokenKind::Punctuation(Punctuation::Hyphen))
352    }
353
354    /// Checks whether the token is whitespace.
355    pub fn is_whitespace(&self) -> bool {
356        matches!(self, TokenKind::Space(_) | TokenKind::Newline(_))
357    }
358
359    pub fn is_upos(&self, upos: UPOS) -> bool {
360        let Some(Some(meta)) = self.as_word() else {
361            return false;
362        };
363
364        meta.pos_tag == Some(upos)
365    }
366
367    // Miscellaneous non-is methods
368
369    /// Checks that `self` is the same enum variant as `other`, regardless of
370    /// whether the inner metadata is also equal.
371    pub fn matches_variant_of(&self, other: &Self) -> bool {
372        self.with_default_data() == other.with_default_data()
373    }
374
375    /// Produces a copy of `self` with any inner data replaced with its default
376    /// value. Useful for making comparisons on just the variant of the
377    /// enum.
378    pub fn with_default_data(&self) -> Self {
379        match self {
380            TokenKind::Word(_) => TokenKind::Word(Default::default()),
381            TokenKind::Punctuation(_) => TokenKind::Punctuation(Default::default()),
382            TokenKind::Number(..) => TokenKind::Number(Default::default()),
383            TokenKind::Space(_) => TokenKind::Space(Default::default()),
384            TokenKind::Newline(_) => TokenKind::Newline(Default::default()),
385            _ => self.clone(),
386        }
387    }
388
389    /// Construct a [`TokenKind::Word`] with no metadata.
390    pub fn blank_word() -> Self {
391        Self::Word(None)
392    }
393
394    // Punctuation and symbol non-is methods
395
396    pub fn as_mut_quote(&mut self) -> Option<&mut Quote> {
397        self.as_mut_punctuation()?.as_mut_quote()
398    }
399
400    pub fn as_quote(&self) -> Option<&Quote> {
401        self.as_punctuation()?.as_quote()
402    }
403}
404
405#[cfg(test)]
406mod tests {
407    use crate::Document;
408
409    #[test]
410    fn car_is_singular_noun() {
411        let doc = Document::new_plain_english_curated("car");
412        let tk = &doc.tokens().next().unwrap().kind;
413        assert!(tk.is_singular_noun());
414    }
415
416    #[test]
417    fn traffic_is_mass_noun_only() {
418        let doc = Document::new_plain_english_curated("traffic");
419        let tk = &doc.tokens().next().unwrap().kind;
420        assert!(tk.is_mass_noun_only());
421    }
422
423    #[test]
424    fn equipment_is_mass_noun() {
425        let doc = Document::new_plain_english_curated("equipment");
426        let tk = &doc.tokens().next().unwrap().kind;
427        assert!(tk.is_mass_noun());
428    }
429
430    #[test]
431    fn equipment_is_non_countable_noun() {
432        let doc = Document::new_plain_english_curated("equipment");
433        let tk = &doc.tokens().next().unwrap().kind;
434        assert!(tk.is_non_countable_noun());
435    }
436
437    #[test]
438    fn equipment_isnt_countable_noun() {
439        let doc = Document::new_plain_english_curated("equipment");
440        let tk = &doc.tokens().next().unwrap().kind;
441        assert!(!tk.is_countable_noun());
442    }
443
444    #[test]
445    fn ate_is_simple_past_only() {
446        let doc = Document::new_plain_english_curated("ate");
447        let tk = &doc.tokens().next().unwrap().kind;
448        assert!(tk.is_verb_simple_past_only());
449        assert!(!tk.is_verb_past_participle_only());
450    }
451
452    #[test]
453    fn eaten_is_past_participle_only() {
454        let doc = Document::new_plain_english_curated("eaten");
455        let tk = &doc.tokens().next().unwrap().kind;
456        assert!(tk.is_verb_past_participle_only());
457        assert!(!tk.is_verb_simple_past_only());
458    }
459
460    #[test]
461    fn thought_is_regular_past_form() {
462        let doc = Document::new_plain_english_curated("thought");
463        let tk = &doc.tokens().next().unwrap().kind;
464        assert!(tk.is_verb_regular_past_form());
465    }
466
467    #[test]
468    fn oov_word_is_oov() {
469        let doc = Document::new_plain_english_curated("nonexistentword");
470        let tk = &doc.tokens().next().unwrap().kind;
471        assert!(tk.is_oov());
472    }
473
474    #[test]
475    fn known_word_is_not_oov() {
476        let doc = Document::new_plain_english_curated("car");
477        let tk = &doc.tokens().next().unwrap().kind;
478        assert!(!tk.is_oov());
479    }
480
481    #[test]
482    fn non_word_tokens_are_not_oov() {
483        let doc = Document::new_plain_english_curated("Hello, world!");
484        let tokens: Vec<_> = doc.tokens().collect();
485
486        // Comma should not be OOV
487        assert!(!tokens[1].kind.is_oov());
488        // Exclamation mark should not be OOV
489        assert!(!tokens[3].kind.is_oov());
490    }
491}
harper_core/token_kind.rs

harper_core/
token_kind.rs
