unobtanium-segmenter 0.5.2

A text segmentation toolbox for search applications inspired by charabia and tantivy.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

// SPDX-FileCopyrightText: 2026 Slatian
//
// SPDX-License-Identifier: LGPL-3.0-only

use whatlang::detect_script;

use crate::SegmentedToken;
use crate::augmentation::Augmenter;

/// Will run just script detection using [whatlang].
#[derive(Debug, Clone, Default)]
pub struct AugmentationDetectScript {}

impl AugmentationDetectScript {
	/// Create a new AugmentationDetectScript instance
	pub fn new() -> Self {
		Default::default()
	}
}

impl Augmenter for AugmentationDetectScript {
	#[allow(clippy::collapsible_if)]
	fn augment<'a>(&self, mut token: SegmentedToken<'a>) -> SegmentedToken<'a> {
		if !token.is_known_word {
			if let Some(script) = detect_script(token.text) {
				token.detected_script = Some(script);
			}
		}
		return token;
	}
}