unobtanium-segmenter 0.5.2

A text segmentation toolbox for search applications inspired by charabia and tantivy.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

// SPDX-FileCopyrightText: 2026 Slatian
//
// SPDX-License-Identifier: LGPL-3.0-only

use crate::SegmentedToken;
use crate::augmentation::Augmenter;

/// Allows easily applying an [Augmenter] to an iterator of tokens.
///
/// It can be used for [augmentation][crate::augmentation] and [normalization][crate::normalization] processing steps.
///
/// It can be used on any iterator that has [SegmentedToken] as its `item`.
pub trait ChainAugmenter<'a, A: Augmenter> {
	/// Process the iterator using an augmenter or normlizer
	fn chain_augmenter(self, augmenter: &A) -> impl Iterator<Item = SegmentedToken<'a>>;

	/// Process the iterator using an owned augmenter or normlizer
	fn chain_owned_augmenter(self, augmenter: A) -> impl Iterator<Item = SegmentedToken<'a>>;
}

impl<'a, A: Augmenter + 'a, I: Iterator<Item = SegmentedToken<'a>>> ChainAugmenter<'a, A> for I
where
	Self: 'a,
{
	fn chain_augmenter(self, augmenter: &A) -> impl Iterator<Item = SegmentedToken<'a>> {
		return self.map(|t| augmenter.augment(t));
	}

	fn chain_owned_augmenter(self, augmenter: A) -> impl Iterator<Item = SegmentedToken<'a>> {
		return self.map(move |t| augmenter.augment(t));
	}
}