uqa-analysis 0.3.6

Tokenizers, char/token filters, and analyzers for UQA full-text search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Transfer native morphology leases while replacing UTF-16 term and token buffers.

use uqa_core::memory::Budgeted;

use crate::nori::NoriOutput;
use crate::token::TokenBatch;
use crate::{AnalysisResult, AnalyzedText, FilteredText};

impl AnalyzedText {
    pub(crate) fn from_nori_budgeted(
        output: Budgeted<NoriOutput>,
        input: &FilteredText<'_>,
        poll: &mut dyn FnMut() -> AnalysisResult<()>,
    ) -> AnalysisResult<Budgeted<Self>> {
        let (output, memory) = output.into_parts();
        super::super::native::output_budgeted(
            TokenBatch {
                tokens: output.tokens,
                final_position_increment: output.final_position_increment,
                terminal: output.terminal,
            },
            output.final_offset_utf16,
            memory,
            input,
            poll,
        )
    }
}