use std::{
fmt::Debug,
hash::{Hash, Hasher},
};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone)]
pub struct Token<T>
where
T: PartialEq + Clone + Debug,
{
normalized: T,
original: String,
pub is_left_joinable: bool,
pub is_right_joinable: bool,
}
impl From<&str> for Token<String> {
fn from(text: &str) -> Self {
Token::new(text.to_owned(), text.to_owned(), true, true)
}
}
impl<T> Token<T>
where
T: PartialEq + Clone + Debug,
{
pub fn new(
normalized: T,
original: String,
is_left_joinable: bool,
is_right_joinable: bool,
) -> Self {
Token {
normalized,
original,
is_left_joinable,
is_right_joinable,
}
}
pub fn original(&self) -> &str {
&self.original
}
pub fn set_normalized(&mut self, normalized: T) {
self.normalized = normalized;
}
pub fn normalized(&self) -> &T {
&self.normalized
}
pub fn get_original_length(&self) -> usize {
self.original.chars().count()
}
}
impl<T> PartialEq for Token<T>
where
T: PartialEq + Clone + Debug,
{
fn eq(&self, other: &Self) -> bool {
self.normalized == other.normalized
}
}
impl<T> Hash for Token<T>
where
T: PartialEq + Clone + Debug + Hash,
{
fn hash<H: Hasher>(&self, state: &mut H) {
self.normalized.hash(state);
}
}