harper_core/
lib.rs

1#![doc = include_str!("../README.md")]
2#![allow(dead_code)]
3
4mod case;
5mod char_ext;
6mod char_string;
7mod currency;
8mod dict_word_metadata;
9mod dict_word_metadata_orthography;
10mod document;
11mod edit_distance;
12pub mod expr;
13mod fat_token;
14mod ignored_lints;
15mod indefinite_article;
16mod irregular_nouns;
17mod irregular_verbs;
18pub mod language_detection;
19mod lexing;
20pub mod linting;
21mod mask;
22mod number;
23mod offsets;
24pub mod parsers;
25pub mod patterns;
26mod punctuation;
27mod regular_nouns;
28mod render_markdown;
29mod span;
30pub mod spell;
31mod sync;
32mod thesaurus_helper;
33mod title_case;
34mod token;
35mod token_kind;
36mod token_string_ext;
37mod vec_ext;
38pub mod weir;
39pub mod weirpack;
40
41use render_markdown::render_markdown;
42use std::collections::{BTreeMap, VecDeque};
43
44pub use case::{Case, CaseIterExt};
45pub use char_string::{CharString, CharStringExt};
46pub use currency::Currency;
47pub use dict_word_metadata::{
48    AdverbData, ConjunctionData, Degree, DeterminerData, Dialect, DialectFlags, DictWordMetadata,
49    NounData, PronounData, VerbData, VerbForm, VerbFormFlags,
50};
51pub use dict_word_metadata_orthography::{OrthFlags, Orthography};
52pub use document::Document;
53pub use fat_token::{FatStringToken, FatToken};
54pub use ignored_lints::{IgnoredLints, LintContext};
55pub use indefinite_article::{InitialSound, starts_with_vowel};
56pub use irregular_nouns::IrregularNouns;
57pub use irregular_verbs::IrregularVerbs;
58use linting::Lint;
59pub use mask::{Mask, Masker, RegexMasker};
60pub use number::{Number, OrdinalSuffix};
61pub use punctuation::{Punctuation, Quote};
62pub use regular_nouns::{get_plurals, get_singulars};
63pub use span::Span;
64pub use sync::{LSend, Lrc};
65pub use title_case::{make_title_case, make_title_case_str};
66pub use token::Token;
67pub use token_kind::TokenKind;
68pub use token_string_ext::TokenStringExt;
69pub use vec_ext::VecExt;
70
71/// Return `harper-core` version
72pub fn core_version() -> &'static str {
73    env!("CARGO_PKG_VERSION")
74}
75
76/// A utility function that removes overlapping lints in a vector,
77/// keeping the more important ones.
78///
79/// Note: this function will change the ordering of the lints.
80pub fn remove_overlaps(lints: &mut Vec<Lint>) {
81    if lints.len() < 2 {
82        return;
83    }
84
85    let mut remove_indices = VecDeque::new();
86    lints.sort_by_key(|l| l.priority);
87    lints.sort_by_key(|l| (l.span.start, !0 - l.span.end));
88
89    let mut cur = 0;
90
91    for (i, lint) in lints.iter().enumerate() {
92        if lint.span.start < cur {
93            remove_indices.push_back(i);
94            continue;
95        }
96        cur = lint.span.end;
97    }
98
99    lints.remove_indices(remove_indices);
100}
101
102/// Remove overlapping lints from a map keyed by rule name, similar to [`remove_overlaps`].
103///
104/// The map is treated as if all contained lints were in a single flat collection, ensuring the
105/// same lint would be kept regardless of whether it originated from `lint` or `organized_lints`.
106pub fn remove_overlaps_map<K: Ord>(lint_map: &mut BTreeMap<K, Vec<Lint>>) {
107    let total: usize = lint_map.values().map(Vec::len).sum();
108    if total < 2 {
109        return;
110    }
111
112    struct IndexedSpan {
113        rule_idx: usize,
114        lint_idx: usize,
115        priority: u8,
116        start: usize,
117        end: usize,
118    }
119
120    let mut removal_flags: Vec<Vec<bool>> = lint_map
121        .values()
122        .map(|lints| vec![false; lints.len()])
123        .collect();
124
125    let mut spans = Vec::with_capacity(total);
126    for (rule_idx, (_, lints)) in lint_map.iter().enumerate() {
127        for (lint_idx, lint) in lints.iter().enumerate() {
128            spans.push(IndexedSpan {
129                priority: lint.priority,
130                rule_idx,
131                lint_idx,
132                start: lint.span.start,
133                end: lint.span.end,
134            });
135        }
136    }
137
138    spans.sort_by_key(|span| span.priority);
139    spans.sort_by_key(|span| (span.start, usize::MAX - span.end));
140
141    let mut cur = 0;
142    for span in spans {
143        if span.start < cur {
144            removal_flags[span.rule_idx][span.lint_idx] = true;
145        } else {
146            cur = span.end;
147        }
148    }
149
150    for (rule_idx, (_, lints)) in lint_map.iter_mut().enumerate() {
151        if removal_flags[rule_idx].iter().all(|flag| !*flag) {
152            continue;
153        }
154
155        let mut idx = 0;
156        lints.retain(|_| {
157            let remove = removal_flags[rule_idx][idx];
158            idx += 1;
159            !remove
160        });
161    }
162}
163
164#[cfg(test)]
165mod tests {
166    use std::hash::DefaultHasher;
167    use std::hash::{Hash, Hasher};
168
169    use itertools::Itertools;
170    use quickcheck_macros::quickcheck;
171
172    use crate::linting::Lint;
173    use crate::remove_overlaps_map;
174    use crate::spell::FstDictionary;
175    use crate::{
176        Dialect, Document,
177        linting::{LintGroup, Linter},
178        remove_overlaps,
179    };
180
181    #[test]
182    fn keeps_space_lint() {
183        let doc = Document::new_plain_english_curated("Ths  tet");
184
185        let mut linter = LintGroup::new_curated(FstDictionary::curated(), Dialect::American);
186
187        let mut lints = linter.lint(&doc);
188
189        dbg!(&lints);
190        remove_overlaps(&mut lints);
191        dbg!(&lints);
192
193        assert_eq!(lints.len(), 3);
194    }
195
196    #[quickcheck]
197    fn overlap_removals_have_equivalent_behavior(s: String) {
198        let doc = Document::new_plain_english_curated(&s);
199        let mut linter = LintGroup::new_curated(FstDictionary::curated(), Dialect::American);
200
201        let mut lint_map = linter.organized_lints(&doc);
202        let mut lint_flat: Vec<_> = lint_map.values().flatten().cloned().collect();
203
204        remove_overlaps_map(&mut lint_map);
205        remove_overlaps(&mut lint_flat);
206
207        let post_removal_flat: Vec<_> = lint_map.values().flatten().cloned().collect();
208
209        fn hash_lint(lint: &Lint) -> u64 {
210            let mut hasher = DefaultHasher::new();
211            lint.hash(&mut hasher);
212            hasher.finish()
213        }
214
215        // We want to ignore ordering, so let us hash these first and sort them.
216        let lint_flat_hashes: Vec<_> = lint_flat.iter().map(hash_lint).sorted().collect();
217        let post_removal_flat_hashes: Vec<_> =
218            post_removal_flat.iter().map(hash_lint).sorted().collect();
219
220        assert_eq!(post_removal_flat_hashes, lint_flat_hashes);
221    }
222}
harper_core/lib.rs

harper_core/
lib.rs
