unobtanium-segmenter 0.5.2

A text segmentation toolbox for search applications inspired by charabia and tantivy.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

// SPDX-FileCopyrightText: 2026 Slatian
//
// SPDX-License-Identifier: LGPL-3.0-only

use std::vec::IntoIter;

use crate::SegmentedToken;
use crate::UseOrSubdivide;
use crate::segmentation::Segmenter;

/// Simplified Interface that assumes one token in, one token out.
///
/// Usable for augmentation and normalization.
pub trait Augmenter {
	/// Apply augmentation function to the given token and return it.
	fn augment<'a>(&self, token: SegmentedToken<'a>) -> SegmentedToken<'a>;
}

impl<T> Segmenter for T
where
	T: Augmenter,
{
	type SubdivisionIter<'a> = IntoIter<SegmentedToken<'a>>;

	fn subdivide<'a>(
		&self,
		token: SegmentedToken<'a>,
	) -> UseOrSubdivide<SegmentedToken<'a>, Self::SubdivisionIter<'a>> {
		return UseOrSubdivide::Use(self.augment(token));
	}
}