1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
use std::fmt;
use crate::TOKEN_SEPARATOR;
/// Handler of a gram.
#[derive(Clone, Copy, Eq)]
pub struct Gram<'a> {
data: &'a [u8],
}
impl<'a> Gram<'a> {
/// Creates a [`Gram`] from a byte slice.
#[inline]
pub const fn new(data: &'a [u8]) -> Self {
Self { data }
}
/// Creates a [`Gram`] from a string.
#[inline]
pub const fn from_str(data: &'a str) -> Self {
Self {
data: data.as_bytes(),
}
}
/// Copies `self` into a new `Vec`.
#[inline]
pub fn to_vec(self) -> Vec<u8> {
self.data.to_vec()
}
/// Gets the reference to the byte slice.
#[inline]
pub const fn raw(&self) -> &[u8] {
self.data
}
/// Pops the last token.
///
/// ```
/// use tongrams::Gram;
///
/// let tokens = "abc de f";
/// let mut gram = Gram::from_str(tokens);
///
/// let (gram, last) = gram.pop_token().unwrap();
/// assert_eq!(gram.raw(), "abc de".as_bytes());
/// assert_eq!(last.raw(), "f".as_bytes());
///
/// let (gram, last) = gram.pop_token().unwrap();
/// assert_eq!(gram.raw(), "abc".as_bytes());
/// assert_eq!(last.raw(), "de".as_bytes());
///
/// assert_eq!(gram.pop_token(), None);
/// ```
#[inline(always)]
pub fn pop_token(&self) -> Option<(Self, Self)> {
let data = self.data;
data.iter()
.rev()
.position(|&x| x == TOKEN_SEPARATOR)
.map(|i| {
let pos = data.len() - i;
let pfx = &data[..pos - 1];
let sfx = &data[pos..];
(Self { data: pfx }, Self { data: sfx })
})
}
/// Splits the gram into tokens.
///
/// ```
/// use tongrams::Gram;
///
/// let tokens = "abc de f";
/// let mut gram = Gram::from_str(tokens);
///
/// let tokens = gram.split_to_tokens();
/// assert_eq!(tokens.len(), 3);
/// assert_eq!(tokens[0].raw(), "abc".as_bytes());
/// assert_eq!(tokens[1].raw(), "de".as_bytes());
/// assert_eq!(tokens[2].raw(), "f".as_bytes());
/// ```
#[inline(always)]
pub fn split_to_tokens(&self) -> Vec<Self> {
self.data
.split(|&b| b == TOKEN_SEPARATOR)
.map(|data| Self { data })
.collect()
}
}
impl<'a> PartialEq for Gram<'a> {
fn eq(&self, other: &Self) -> bool {
self.data == other.data
}
}
impl<'a> fmt::Display for Gram<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", String::from_utf8(self.to_vec()).unwrap())
}
}
impl<'a> fmt::Debug for Gram<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let data = String::from_utf8(self.data.to_vec()).unwrap();
f.debug_struct("Gram").field("data", &data).finish()
}
}