unobtanium_segmenter/

segmented_token.rs

use whatlang::Lang;
use whatlang::Script;

/// The main representation of data this crate works on.
///
/// A token is effectively `text` with metadata attached, this struct being the metadata carrier.
#[derive(Debug, Clone, PartialEq)]
pub struct SegmentedToken<'a> {
	/// The piece of text that this token represents.
	///
	/// This should be borrowed from the initial text that was fed to the segmenter chain.
	pub text: &'a str,

	/// If a [normalizer](crate::normalization) output was different from `text` the result will be stored in here.
	///
	/// It is recommended that you fetch normalized text using [get_text_prefer_normalized()][Self::get_text_prefer_normalized] or [get_text_prefer_normalized_owned()][Self::get_text_prefer_normalized_owned].
	pub normalized_text: Option<String>,

	/// What kind of token this is.
	///
	/// Set by:
	/// * [NaiveWordSplitter][crate::segmentation::NaiveWordSplitter]
	/// * [AugmentationClassify][crate::augmentation::AugmentationClassify]
	pub kind: Option<SegmentedTokenKind>,

	/// The primary script as detected by a script or language detection augmenter.
	///
	/// Information about detected scripts is inherited across splitting.
	pub detected_script: Option<Script>,

	/// The primary language detected by a language detection augmenter.
	///
	/// Information about detected languages in inherited across splitting.
	pub detected_language: Option<Lang>,

	/// How confident the language detector was about the language that it detectd.
	///
	/// This scales inbetween `0` (not confident at all) and `1` (most confident).
	pub detected_language_confidence: f64,

	/// Wheter the language detector considers its output to be reliable.
	pub is_detected_language_relible: bool,

	/// Indicates that no further splitting is neccessary.
	///
	/// This should be set to true if the token was a valid word in a dictionary.
	pub is_known_word: bool,

	/// Indicates that this token marks the end of a sentence.
	///
	/// This should only be set on tokens with an empty `text` field. It is not inherited.
	pub is_end_of_sentence: bool,
	//TODO: Add useful flags here
	// contains_emoji: bool,
}

impl<'a> SegmentedToken<'a> {
	/// Create a segmented token from scratch. (You likely won't need it)
	///
	/// If you are wwriting a segmenter have a look at [new_derived_from()][Self::new_derived_from].
	///
	/// For creating the initial token consider usng the `From` implementations or the [StartSegmentationChain][crate::chain::StartSegmentationChain] trait.
	pub fn new(text: &'a str, kind: Option<SegmentedTokenKind>) -> Self {
		let mut out = Self {
			text,
			kind,
			normalized_text: None,
			is_known_word: false,
			detected_script: None,
			detected_language: None,
			detected_language_confidence: 0.0,
			is_detected_language_relible: false,
			is_end_of_sentence: false,
		};
		match kind {
			Some(SegmentedTokenKind::AlphaNumeric) => {
				out.detected_script = whatlang::detect_script(text);
			}
			_ => { /* Do nothing */ }
		}
		return out;
	}

	/// Create a token with a given text that inerits metadata from the `from` token.
	///
	/// This is the recommended constructor to use inside a segmenter after splitting.
	pub fn new_derived_from(text: &'a str, from: &Self) -> Self {
		Self {
			text,
			kind: None,
			is_known_word: from.is_known_word,
			detected_script: from.detected_script,
			detected_language: from.detected_language,
			detected_language_confidence: from.detected_language_confidence,
			is_detected_language_relible: from.is_detected_language_relible,
			normalized_text: None,
			is_end_of_sentence: false,
		}
	}

	/// Create a new token that carries an `is_end_of_sentence` marker.
	///
	/// Recommended way of deriving the empty text:
	/// ```rust
	/// # use unobtanium_segmenter::SegmentedToken;
	/// # let token = SegmentedToken::new("Some example sentence to segment.", None);
	/// # let sentence = token.text; // Actual segmenter goes here
	/// let (main, tail) = sentence.split_at(sentence.len());
	/// SegmentedToken::new_derived_from(main, &token);
	/// SegmentedToken::new_end_of_sentence(tail);
	/// ```
	pub fn new_end_of_sentence(empty_text: &'a str) -> Self {
		let mut new = Self::new(empty_text, None);
		new.is_end_of_sentence = true;
		return new;
	}

	/// Helper function to convert texts that came ot of a simple helper function
	/// back into segments.
	///
	/// Using this implies that further segmenting didn't change anything
	/// for the metadta of the child segments.
	pub fn covert_to_child_segements_of_self(
		&'a self,
		texts: &'a [&'a str],
	) -> impl Iterator<Item = SegmentedToken<'a>> + 'a {
		texts.iter().map(|text| Self::new_derived_from(text, self))
	}

	/// Builder like convenience function to set the `is_known_word` flag.
	#[inline(always)]
	pub fn with_is_kown_word(mut self, is_known_word: bool) -> Self {
		self.is_known_word = is_known_word;
		return self;
	}

	/// Return the `normalized_text` of this token if present and `text` if not as a `str`.
	pub fn get_text_prefer_normalized(&self) -> &str {
		if let Some(normalized_text) = &self.normalized_text {
			return normalized_text.as_str();
		} else {
			return self.text;
		}
	}

	/// This is the same as [get_text_prefer_normalized()][Self::get_text_prefer_normalized], but returns an owned String instead.
	pub fn get_text_prefer_normalized_owned(&self) -> String {
		self.get_text_prefer_normalized().to_string()
	}
}

impl<'a> From<&'a String> for SegmentedToken<'a> {
	fn from(value: &'a String) -> Self {
		Self::new(value, None)
	}
}

impl<'a> From<&'a str> for SegmentedToken<'a> {
	fn from(value: &'a str) -> Self {
		Self::new(value, None)
	}
}

/// What kind of content to expect from a [SegmentedToken].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SegmentedTokenKind {
	/// The token is a collection of any kind of letters and numbers
	AlphaNumeric,

	/// The token is some kind of seperator
	Separator,

	/// The token represents a symbol
	Symbol,
}