unobtanium-segmenter 0.5.2

A text segmentation toolbox for search applications inspired by charabia and tantivy.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

// SPDX-FileCopyrightText: 2026 Slatian
//
// SPDX-License-Identifier: LGPL-3.0-only

use crate::SegmentedToken;
use crate::SegmentedTokenKind;
use crate::augmentation::Augmenter;

/// Will lowercase anything that can be lowercased using the rust builtin lowercasing methods.
///
/// This will skip the token if the token kind indicates that the token doesn't contain any letters to lowercase.
#[derive(Debug, Clone, Default)]
pub struct NormalizationLowercase {/* Nothing in here */}

impl NormalizationLowercase {
	/// Create a new NormalizationLowercase instance.
	pub fn new() -> Self {
		Default::default()
	}
}

impl Augmenter for NormalizationLowercase {
	fn augment<'a>(&self, mut token: SegmentedToken<'a>) -> SegmentedToken<'a> {
		if matches!(token.kind, Some(SegmentedTokenKind::AlphaNumeric) | None) {
			token.update_normalized_string(token.get_text_prefer_normalized().to_lowercase(), None);
		}
		return token;
	}
}