quickner-core 0.0.1-alpha.20

A fast and simple NER tool
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

// quickner
//
// NER tool for quick and simple NER annotation
// Copyright (C) 2023, Omar MHAIMDAT
//
// Licensed under Mozilla Public License 2.0
//
use std::{collections::HashSet, str};

use indicatif::{ProgressBar, ProgressStyle};

/// Checks if a string is alphanumeric.
/// # Examples
/// ```
/// use utils::is_alphanumeric;
/// let text = "Hello, world!";
/// assert_eq!(is_alphanumeric(text), true);
/// ```
pub(crate) fn is_alphanumeric(text: &str) -> bool {
    if text.is_empty() {
        return false;
    }
    text.chars().all(|c| c.is_alphanumeric())
}

/// Checks if a string contains punctuation.
/// # Examples
/// ```
/// use utils::contains_punctuation;
/// let text = "Hello, world!";
/// assert_eq!(contains_punctuation(text), true);
/// ```
pub(crate) fn contains_punctuation(text: &str) -> bool {
    if text.is_empty() {
        return false;
    }
    text.chars().any(|c| c.is_ascii_punctuation())
}

/// Checks if a string contains numbers.
/// # Examples
/// ```
/// use utils::contains_numbers;
/// let text_without = "Hello, world!";
/// assert_eq!(contains_numbers(text), false);
/// let text_with = "Hello, 123!";
/// assert_eq!(contains_numbers(text), true);
/// ```
/// # Panics
/// Panics if the string contains non-ASCII characters.
/// # Errors
/// Returns an error if the string contains non-ASCII characters.
pub(crate) fn contains_numbers(text: &str) -> bool {
    if text.is_empty() {
        return false;
    }
    text.chars().any(|c| c.is_ascii_digit())
}

/// Checks if a string contains special characters.
/// # Examples
/// ```
/// use utils::contains_special_characters;
/// let text_without = "Hello, world!";
/// assert_eq!(contains_special_characters(text), false);
/// let text_with = "Hello, world@!";
/// assert_eq!(contains_special_characters(text), true);
/// ```
/// # Panics
/// Panics if the string contains non-ASCII characters.
/// # Errors
/// Returns an error if the string contains non-ASCII characters.
pub(crate) fn contains_special_characters(text: &str, special_characters: HashSet<char>) -> bool {
    if text.is_empty() {
        return false;
    }
    text.chars().any(|c| special_characters.contains(&c))
}

/// Get a progress bar with a custom style.
/// # Examples
/// ```
/// use utils::get_progress_bar;
/// let progress_bar = get_progress_bar(100);
/// ```
pub(crate) fn get_progress_bar(total: u64) -> ProgressBar {
    let progress_bar = ProgressBar::new(total);

    progress_bar.set_style(ProgressStyle::with_template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.green/blue}] {human_pos}/{human_len} ({eta})")
        .unwrap()
        .progress_chars("##-"));
    progress_bar
}

pub fn hash_string(text: &str) -> String {
    use std::collections::hash_map::DefaultHasher;
    use std::hash::{Hash, Hasher};

    let mut hasher = DefaultHasher::new();
    text.hash(&mut hasher);
    format!("{:x}", hasher.finish())
}

pub(crate) fn is_valid_utf8(text: &str) -> bool {
    let b = text.as_bytes();
    if str::from_utf8(b).is_err() {
        return false;
    } else {
        return true;
    }
}

pub(crate) fn char_to_byte(text: String, start: usize, end: usize) -> (usize, usize) {
    let start = text.char_indices().nth(start);
    let end = text.char_indices().nth(end);
    let start = match start {
        Some(start) => start.0,
        None => 0,
    };
    let end = match end {
        Some(end) => end.0,
        None => text.len(),
    };
    (start, end)
}