unobtanium-segmenter 0.5.2

A text segmentation toolbox for search applications inspired by charabia and tantivy.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

// SPDX-FileCopyrightText: 2026 Slatian
//
// SPDX-License-Identifier: LGPL-3.0-only

use crate::SegmentedToken;
use crate::SubdivisionMap;
use crate::segmentation::Segmenter;

/// Allows easily applying a [Segmenter] to an iterator of tokens.
///
/// If your Segmenter is also an [Augmenter][crate::augmentation::Augmenter] please use [ChainAugmenter][crate::chain::ChainAugmenter] instead.
///
/// It can be used on any iterator that has [SegmentedToken] as its `item`.
pub trait ChainSegmenter<'a, S: Segmenter>
where
	Self: 'a,
{
	/// Process the iterator using a segmenter
	fn chain_segmenter(self, segmenter: &S) -> impl Iterator<Item = SegmentedToken<'a>>;

	/// Process the iterator using a segmenter
	fn chain_owned_segmenter(self, segmenter: S) -> impl Iterator<Item = SegmentedToken<'a>>;
}

impl<'a, S: Segmenter + 'a, I: Iterator<Item = SegmentedToken<'a>>> ChainSegmenter<'a, S> for I
where
	Self: 'a,
{
	fn chain_segmenter(self, segmenter: &S) -> impl Iterator<Item = SegmentedToken<'a>> {
		return SubdivisionMap::new(self, |t| segmenter.subdivide(t));
	}

	fn chain_owned_segmenter(self, segmenter: S) -> impl Iterator<Item = SegmentedToken<'a>> {
		return SubdivisionMap::new(self, move |t| segmenter.subdivide(t));
	}
}