1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

#![cfg_attr(test, feature(test))]

use unicode_categories::UnicodeCategories;

#[cfg(feature = "censor")]
pub(crate) mod buffer_proxy_iterator;
#[cfg(feature = "censor")]
pub(crate) mod censor;
#[cfg(feature = "censor")]
pub(crate) mod feature_cell;
#[cfg(feature = "censor")]
pub(crate) mod mtch;
#[cfg(feature = "censor")]
pub(crate) mod trie;
#[cfg(feature = "censor")]
pub(crate) mod typ;

#[cfg(feature = "context")]
pub(crate) mod context;

#[cfg(feature = "width")]
pub(crate) mod width;

#[cfg(feature = "width")]
pub use width::{trim_to_width, width, width_str};

#[cfg(feature = "censor")]
pub use typ::Type;

#[cfg(feature = "censor")]
pub use censor::{Censor, CensorIter, CensorStr};

// Facilitate experimentation with different hash collections.
#[cfg(feature = "censor")]
pub(crate) type Map<K, V> = rustc_hash::FxHashMap<K, V>;

#[cfg(feature = "censor")]
pub(crate) type Set<V> = rustc_hash::FxHashSet<V>;

#[cfg(feature = "customize")]
pub use censor::add_word;

#[cfg(feature = "context")]
pub use context::{
    BlockReason, Context, ContextProcessingOptions, ContextRateLimitOptions,
    ContextRepetitionLimitOptions,
};

/// Trims whitespace characters from both ends of a string, according to the definition of
/// `crate::is_whitespace`.
pub fn trim_whitespace(s: &str) -> &str {
    // Some characters are effectively whitespace if they are at the beginning of a string.
    // https://www.compart.com/en/unicode/U+0488
    // https://www.compart.com/en/unicode/U+0489
    s.trim_start_matches(|c| is_whitespace(c) || matches!(c, '\u{0488}' | '\u{0489}'))
        .trim_end_matches(is_whitespace)
}

/// Returns true iff the character is effectively whitespace. The definition of whitespace is broader
/// than that of Unicode, because it includes control characters and a few additional blank characters.
pub fn is_whitespace(c: char) -> bool {
    // NOTE: The following characters are not detected by standard means but show up as blank.

    // https://www.compart.com/en/unicode/U+115F
    // https://www.compart.com/en/unicode/U+1160
    // https://www.compart.com/en/unicode/U+2800
    // https://www.compart.com/en/unicode/U+3164
    // https://www.compart.com/en/unicode/U+FFA0
    c.is_whitespace()
        || c.is_other()
        || matches!(
            c,
            '\u{115F}' | '\u{1160}' | '\u{2800}' | '\u{3164}' | '\u{FFA0}'
        )
}

#[cfg(test)]
mod tests {
    #![allow(unused_imports)]
    extern crate test;

    #[test]
    fn trim_whitespace() {
        // General.
        assert_eq!(crate::trim_whitespace("\u{0020}\u{00A0}\u{2000}\u{2001}\u{2002}\u{2004}\u{2005}\u{2006}\u{2007}\u{2008}\u{2009}\u{200A}\u{200B}\u{200C}\u{200D}\u{2028}\u{205F}\u{3000}"), "");

        // Extra cases.
        assert_eq!(
            crate::trim_whitespace(" \u{1160} \u{2800} abc \u{3164} \u{FFA0} \t \u{115F} \n "),
            "abc"
        );

        // Special cases.
        assert_eq!(
            crate::trim_whitespace(
                "\u{0488}\u{1160}\u{0489}\u{1160}\u{0488}\u{1160}\u{0489}abc\u{0488}\u{0489}"
            ),
            "abc\u{0488}\u{0489}"
        )
    }

    #[test]
    fn is_whitespace() {
        assert!(crate::is_whitespace(' '));
        assert!(crate::is_whitespace('\u{2800}'));
        assert!(!crate::is_whitespace('a'));
    }
}

doc_comment::doctest!("../README.md");