uqa_analysis/porter/
allocation.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Stemmer scratch and result buffers share the caller's allocation allowance.
8
9use uqa_core::memory::{Budgeted, BudgetedString, BudgetedVec, MemoryBudget, MemoryError};
10
11use super::{algorithm, word::Word, Character};
12use crate::{AnalysisResult, TokenTerm};
13
14pub fn stem(word: &str) -> String {
15    stem_budgeted(word, &MemoryBudget::new(usize::MAX), || Ok(()))
16        .expect("unbounded Porter stemming")
17        .into_parts()
18        .0
19}
20
21#[cfg(test)]
22pub(crate) fn stem_utf16(word: &[u16]) -> Vec<u16> {
23    stem_utf16_budgeted(word, &MemoryBudget::new(usize::MAX), &mut || Ok(()))
24        .expect("unbounded lossless Porter stemming")
25        .into_parts()
26        .0
27}
28
29/// Stem scalar text with reserved scratch/output buffers and cancellation checks.
30///
31/// Input is borrowed. The result retains its string reservation; scratch is released before return. Character loading, prefix scans, suffix stages, and encoding poll for cancellation. Consonant state for repeated `y` is computed without recursion.
32///
33/// ```
34/// use uqa_analysis::porter::stem_budgeted;
35/// use uqa_core::memory::MemoryBudget;
36/// let budget = MemoryBudget::new(4096);
37/// let result = stem_budgeted("relational", &budget, || Ok(()))?;
38/// assert_eq!(&**result, "relat");
39/// assert_eq!(budget.used(), result.reserved_bytes());
40/// assert!(budget.peak() > budget.used());
41/// drop(result);
42/// assert_eq!(budget.used(), 0);
43/// # Ok::<(), uqa_analysis::AnalysisError>(())
44/// ```
45pub fn stem_budgeted(
46    input: &str,
47    budget: &MemoryBudget,
48    mut poll: impl FnMut() -> AnalysisResult<()>,
49) -> AnalysisResult<Budgeted<String>> {
50    let mut word = Word::new(input.chars().map(Character::from), budget, &mut poll)?;
51    algorithm::apply(&mut word)?;
52    let mut length = 0usize;
53    for index in 0..word.len() {
54        if index % 1024 == 0 {
55            word.check()?;
56        }
57        let character = char::from_u32(word[index].0).expect("scalar input and ASCII suffixes");
58        length = length
59            .checked_add(character.len_utf8())
60            .ok_or(MemoryError::SizeOverflow)?;
61    }
62    let mut output = BudgetedString::new(budget);
63    output.reserve(length)?;
64    for index in 0..word.len() {
65        if index % 1024 == 0 {
66            word.check()?;
67        }
68        output.push(char::from_u32(word[index].0).expect("scalar input and ASCII suffixes"))?;
69    }
70    word.check()?;
71    drop(word);
72    let (output, memory) = output.into_parts();
73    Ok(Budgeted::new(output, memory))
74}
75
76/// Stem a complete lossless term, including isolated surrogate elements.
77pub fn stem_term_budgeted(
78    input: &TokenTerm,
79    budget: &MemoryBudget,
80    mut poll: impl FnMut() -> AnalysisResult<()>,
81) -> AnalysisResult<Budgeted<TokenTerm>> {
82    if let Some(input) = input.as_str() {
83        let (term, memory) = stem_budgeted(input, budget, poll)?.into_parts();
84        Ok(Budgeted::new(term.into(), memory))
85    } else {
86        let output = stem_utf16_budgeted(&input.utf16(), budget, &mut poll)?;
87        TokenTerm::from_utf16_budgeted(output, poll)
88    }
89}
90
91pub(super) fn stem_utf16_budgeted(
92    input: &[u16],
93    budget: &MemoryBudget,
94    poll: &mut dyn FnMut() -> AnalysisResult<()>,
95) -> AnalysisResult<Budgeted<Vec<u16>>> {
96    let characters = char::decode_utf16(input.iter().copied()).map(|value| {
97        value.map_or_else(
98            |error| Character(u32::from(error.unpaired_surrogate())),
99            Character::from,
100        )
101    });
102    let mut word = Word::new(characters, budget, poll)?;
103    algorithm::apply(&mut word)?;
104    let mut length = 0usize;
105    for index in 0..word.len() {
106        if index % 1024 == 0 {
107            word.check()?;
108        }
109        let units = char::from_u32(word[index].0).map_or(1, char::len_utf16);
110        length = length.checked_add(units).ok_or(MemoryError::SizeOverflow)?;
111    }
112    let mut output = BudgetedVec::new(budget);
113    output.reserve(length)?;
114    for index in 0..word.len() {
115        if index % 1024 == 0 {
116            word.check()?;
117        }
118        if let Some(character) = char::from_u32(word[index].0) {
119            for unit in character.encode_utf16(&mut [0; 2]) {
120                output.push(*unit)?;
121            }
122        } else {
123            output.push(u16::try_from(word[index].0).expect("isolated surrogate"))?;
124        }
125    }
126    word.check()?;
127    drop(word);
128    let (output, memory) = output.into_parts();
129    Ok(Budgeted::new(output, memory))
130}
uqa_analysis/porter/allocation.rs

uqa_analysis/porter/
allocation.rs
