uqa-analysis 0.3.6

Tokenizers, char/token filters, and analyzers for UQA full-text search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Common-token filter mutation retains exact source projection and replacement leases.

use super::AnalysisToken;
use crate::morphology::filter::{covering_range, ComposingToken};
use crate::morphology::filter::{FilterToken, Work};
use crate::source::SourceProjection;
use crate::{AnalysisResult, TokenTerm};
use std::ops::Range;
use std::sync::Arc;
use uqa_core::memory::MemoryBudget;
use uqa_core::memory::{Budgeted, MemoryReservation};

impl FilterToken for AnalysisToken {
    type Context = Arc<Budgeted<SourceProjection>>;

    fn term(&self) -> impl Iterator<Item = u16> + '_ {
        self.term.utf16_units()
    }
    fn term_len(&self, work: &mut Work<'_>) -> AnalysisResult<usize> {
        if self.term.as_str().is_none() {
            return Ok(self.term.utf16_len());
        }
        let mut length = 0;
        for _ in self.term.utf16_units() {
            work.tick()?;
            length += 1;
        }
        Ok(length)
    }
    fn replace_term(
        &mut self,
        term: Budgeted<Vec<u16>>,
        memory: &mut MemoryReservation,
        context: &Self::Context,
        work: &mut Work<'_>,
    ) -> AnalysisResult<()> {
        let term = TokenTerm::from_utf16_budgeted(term, &mut *work.poll)?;
        let verbatim = if let Some(offsets) = &self.offsets {
            context.is_verbatim_with_control(&term, offsets, work.poll)?
        } else {
            false
        };
        if !self.term.eq_with_control(&term, work.poll)? {
            let old_bytes = self.term.allocation_bytes();
            let (term, allocation) = term.into_parts();
            drop(std::mem::replace(&mut self.term, term));
            drop(memory.split(old_bytes));
            memory.absorb(allocation);
        }
        self.verbatim = verbatim;
        Ok(())
    }
    fn refresh_context(
        &mut self,
        context: &Self::Context,
        work: &mut Work<'_>,
    ) -> AnalysisResult<()> {
        self.verbatim = if let Some(offsets) = &self.offsets {
            context.is_verbatim_with_control(&self.term, offsets, work.poll)?
        } else {
            false
        };
        Ok(())
    }
    fn keyword(&self) -> bool {
        self.keyword
    }
}

impl ComposingToken for AnalysisToken {
    type Span = Option<Range<usize>>;
    fn clone_reserved(
        &self,
        budget: &MemoryBudget,
        work: &mut Work<'_>,
    ) -> AnalysisResult<Budgeted<Self>> {
        self.clone_budgeted(budget, &mut *work.poll)
    }
    fn position_length(&self) -> u32 {
        self.position_length
    }
    fn span(&self) -> Self::Span {
        self.filtered_utf16.clone()
    }
    fn cover(
        &mut self,
        first: &Self::Span,
        last: &Self::Span,
        context: &Self::Context,
        work: &mut Work<'_>,
    ) -> AnalysisResult<()> {
        self.filtered_utf16 = first
            .as_ref()
            .zip(last.as_ref())
            .map(|(first, last)| covering_range(first, last))
            .transpose()?;
        self.offsets = self
            .filtered_utf16
            .clone()
            .map(|range| context.project_with_control(range, work.poll))
            .transpose()?;
        self.refresh_context(context, work)
    }
}