hashtag_regex/

lib.rs

use lazy_static::lazy_static;

// The unicode standard for regexes specified three valid starting characters:
// The ascii '#' (\u{23}), and the two code points '﹟' (\u{FE5F}) and '＃' (\u{FF03}).
// In the following we see two elements showing up multiple times:
// The `HASHES_RE` constant holds the three hashes permitted,
// while the `CONTINUATION_CHAR_RE` defines the class of characters permitted
// in the tag portion of the hashtag.

/// The three kinds of hash allowed by the unicode standard.
///
/// All three of them are escaped even though they're printable.
/// This is because I'm using multi-line regex syntax and that
/// uses the `'#'` character for comments.
///
/// This can be useful when checking that a text does _not_ contain a hash,
/// especially when checking emojis.
///
/// Note that some emojis contain hash code points:
///
/// # Examples
///
/// ```
/// # use regex::Regex;
/// # use hashtag_regex::HASHES_RE_STR;
/// let hash_re = Regex::new(HASHES_RE_STR).unwrap();
/// let input = "#️⃣";
/// // This is the same string:
/// let input_escaped = "\u{1b}\u{5b}\u{32}\u{30}\u{30}\u{7e}\u{23}\u{fe0f}\u{20e3}";
/// // Note the '\u{23}' which is the code point of '#'
/// assert!(input.contains("#"));
/// assert!(hash_re.is_match(&input));
/// assert!(hash_re.is_match(&input_escaped));
/// ```
pub const HASHES_RE_STR: &str = r"[\u{23}\u{FE5F}\u{FF03}]";

lazy_static! {
    /// The characters that make up the actual tag of the hash-tag.
    ///
    /// This set of "continuation characters" is taken directly from the unicode
    /// standard. Note that this _excludes_ the hash symbols, which is important
    /// to keep in mind when matching tags with unicode symbols since some
    /// emojis include hash code points.
    static ref CONTINUATION_CHAR_RE_STRING: String = format!(
        r"[\p{{XID_Continue}}\p{{Extended_Pictographic}}\p{{Emoji_Component}}[-+_]--{}]",
        HASHES_RE_STR
    );

    /// Regex for the complete set of hashtags accoring to the unicode standard.
    /// See the comments inside the definition
    ///
    /// This uses a `(^|[^CONTINUATION_CHAR_RE])` construct in the beginning since
    /// the regex engine does not support look-behind constructs. Luckily, all we
    /// want to know is that there is no continuation character just before the hash,
    /// and we can accomplish that with a simple
    ///
    /// # Examples
    ///
    /// ```
    /// # use regex::Regex;
    /// let hashtag_re = Regex::new(&hashtag_regex::HASHTAG_RE_STRING).unwrap();
    /// let text = "Hello #🌍, wassup? Check out this #regex magic!";
    /// let all_captures: Vec<regex::Captures> = hashtag_re.captures_iter(text).collect();
    /// assert_eq!(all_captures.len(), 2);
    /// assert_eq!(all_captures.get(0).unwrap().name("hashtag").unwrap().as_str(), "#🌍");
    /// assert_eq!(all_captures.get(1).unwrap().name("tag").unwrap().as_str(), "regex");
    /// ```
    pub static ref HASHTAG_RE_STRING: String = format!(r#"(?x)
        (
            # having the string start "character" just before the hash is okay
            ^
            |
            # otherwise, the only requirement is that the preceding character is
            # NOT a continuation character
            [^{}]
        )
        # The actual hashtag: a hash, then at least one tag char. No magic here! :)
        (?P<hashtag>
            (?P<hash>{})
            (?P<tag>{}+)
        )
    "#,
        CONTINUATION_CHAR_RE_STRING.as_str(),
        HASHES_RE_STR,
        CONTINUATION_CHAR_RE_STRING.as_str()
    );
}

#[cfg(test)]
mod tests {
    use itertools::Itertools;
    use regex::Regex;
    use test_case::test_case;

    use super::*;

    lazy_static! {
        // matches any hashtag anywhere in the text
        // useful for find/replace operations
        pub static ref HASHTAG_RE: Regex = Regex::new(&HASHTAG_RE_STRING).unwrap();

        // matches a hashtag _exactly_
        pub static ref HASHTAG_RE_EXACT: Regex = Regex::new(&format!(
            "^{}$",
            HASHTAG_RE_STRING.as_str(),
        ))
        .unwrap();
    }

    #[test]
    fn can_match_any_single_emoji_without_hashtag() {
        let re_string = r"[\p{Extended_Pictographic}\p{Emoji_Component}]";
        let single_emoji_regex = Regex::new(re_string).unwrap();
        let results: Vec<bool> = emojic::grouped::all_variants()
            .flatten()
            .map(|e| single_emoji_regex.is_match(e.grapheme))
            .collect();
        let matched = results.iter().map(|&b| b as u32).sum::<u32>();
        println!(
            "Out of {} emojis, manged to match {}",
            results.len(),
            matched
        );
        assert_ne!(matched, 0, "Not a single emoji got matched?!");
    }

    #[test]
    fn does_not_match_hash_without_tag() {
        for starting_char in &['#', '﹟', '＃'] {
            let input = format!("Hello world, {} alone is not a hashtag!", starting_char);
            assert!(
                !HASHTAG_RE.is_match(&input),
                "Wrongly matched \"{}\"",
                input
            );
        }
    }

    #[test_case("revolution"; "simple ascii hashtag")]
    #[test_case("神key"; "combined non-ascii and ascii hashtag")]
    #[test_case("key神"; "combined ascii and non-ascii hashtag")]
    #[test_case("hello🌍"; "combined ascii and emoji hashtag")]
    #[test_case("🌍domination"; "combined emoji and ascii hashtag")]
    #[test_case("🍕"; "In honour of ~dtBy's ceaseless pizza posting")]
    fn test_hash_and_tag_get_found(tag: &str) -> Result<(), &'static str> {
        for hash in &['#', '﹟', '＃'] {
            let input = format!(
                "See this hashtag I just received {}{}. I have no idea what it means...",
                hash, tag
            );
            let all_captures: Vec<regex::Captures> =
                HASHTAG_RE.captures_iter(&input).into_iter().collect();
            assert_eq!(all_captures.len(), 1);
            let single_capture = all_captures
                .get(0)
                .ok_or("this shouldn't happen due to previous assert")?;

            let capture_match = single_capture
                .name("hash")
                .ok_or("expected to capture hash symbol")?;
            assert_eq!(capture_match.as_str(), hash.to_string());

            let capture_match = single_capture
                .name("tag")
                .ok_or("expected to capture the actual tag")?;
            assert_eq!(capture_match.as_str(), tag);

            let capture_match = single_capture
                .name("hashtag")
                .ok_or("expected to capture full hashtag")?;
            assert_eq!(capture_match.as_str(), format!("{}{}", hash, tag));
        }
        Ok(())
    }

    #[test_case("This is an example #text with tho #hashtags in it", vec!["text", "hashtags"]; "Two simple hashtags")]
    #[test_case("Giving it a little #try#with#three consecutive tags", vec!["try"]; "Don't match hashtags just after tag")]
    fn finding_hashtags(text: &str, expected_tags: Vec<&str>) -> Result<(), &'static str> {
        let all_captures: Vec<regex::Captures> = HASHTAG_RE.captures_iter(text).collect();
        assert_eq!(all_captures.len(), expected_tags.len());
        for (single_capture, expected_tag) in all_captures.into_iter().zip(expected_tags) {
            let capture_match = single_capture
                .name("tag")
                .ok_or("expected to capture the actual tag")?;
            assert_eq!(capture_match.as_str(), expected_tag);
        }
        Ok(())
    }

    #[test]
    fn double_hash_is_no_hashtag() {
        for hash_1 in &['#', '﹟', '＃'] {
            for hash_2 in &['#', '﹟', '＃'] {
                let input = format!("This here: {}{} is not a hashtag!", hash_1, hash_2);
                assert!(
                    !HASHTAG_RE.is_match(&input),
                    r#"Wrongly matched "{}" aka "{}""#,
                    input,
                    input.escape_unicode(),
                );
            }
        }
    }

    #[test]
    fn can_match_every_single_emoji_without_hashtag() {
        let continuation_re =
            Regex::new(&format!("^{}+$", CONTINUATION_CHAR_RE_STRING.as_str())).unwrap();
        let hashes_re = Regex::new(HASHES_RE_STR).unwrap();
        for emoji in emojic::grouped::all_variants().flatten().filter_map(|e| {
            if !hashes_re.is_match(e.grapheme) {
                Some(e.grapheme)
            } else {
                None
            }
        }) {
            assert!(
                continuation_re.is_match(emoji),
                r#"Could not match "{}" aka "{}" as an emoji"#,
                emoji,
                emoji.escape_unicode(),
            );
        }
        println!(
            "Successfully matched {} emojis.",
            emojic::grouped::all_variants().flatten().count()
        )
    }

    #[test]
    fn full_re_matches_each_emoji_grapheme_with_each_hash() {
        let hashes_re = Regex::new(HASHES_RE_STR).unwrap();
        let all_graphemes: Vec<&'static str> = emojic::grouped::all_variants()
            .flatten()
            // .filter(|e| !non_hash_re.is_match(*e.grapheme))
            .filter_map(|e| {
                if !hashes_re.is_match(e.grapheme) {
                    Some(e.grapheme)
                } else {
                    None
                }
            })
            .collect();
        println!("Will check {} combinations", 3 * all_graphemes.len());
        for emoji in all_graphemes.to_owned() {
            for (i, starting_char) in vec!['#', '﹟', '＃'].iter().enumerate() {
                let input = format!("{}{}", starting_char, &emoji);
                assert!(
                    HASHTAG_RE_EXACT.is_match(&input),
                    "Input: \"{}\" aka \"{}\" (hash no. {})",
                    input,
                    input.escape_unicode(),
                    i
                );
            }
        }
    }

    #[test]
    fn full_re_matches_any_two_emojis_as_tag() -> Result<(), String> {
        let hashes_re = Regex::new(HASHES_RE_STR).unwrap();
        let all_pairs: Vec<Vec<&'static str>> = emojic::grouped::all_variants()
            .flatten()
            // .filter(|e| !non_hash_re.is_match(*e.grapheme))
            .filter_map(|e| {
                if !hashes_re.is_match(e.grapheme) {
                    Some(e.grapheme)
                } else {
                    None
                }
            })
            .combinations(2)
            .collect();
        println!(
            "Will try to match {} hashtags with two emojis",
            all_pairs.len()
        );
        for v in all_pairs {
            if v.len() != 2 {
                return Err(format!(
                    "should be length 2, but is length {}",
                    v.len().to_string()
                ));
            }
            let e1 = *v.get(0).ok_or("wtf")?;
            let e2 = *v.get(1).ok_or("wtf")?;
            let tag = format!("{}{}", e1, e2);
            let input = format!("#{}", &tag);
            assert!(
                HASHTAG_RE_EXACT.is_match(&input),
                r#"Failed to match "{}" aka "{}" comprised of "{}" and "{}""#,
                input,
                input.escape_unicode(),
                e1,
                e2
            );
        }
        Ok(())
    }
}