uqa_analysis/term/
allocation.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Canonical term conversion retains both encodings while they coexist.
8
9use uqa_core::memory::{Budgeted, BudgetedString, BudgetedVec, MemoryBudget, MemoryError};
10
11use super::{Representation, TokenTerm};
12use crate::AnalysisResult;
13
14/// The same character boundary in the term's stored representation and UTF-16.
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub(crate) struct TermBoundary {
17    pub offset: usize,
18    pub utf16: usize,
19}
20
21pub(crate) enum TermBuffer {
22    Unicode(BudgetedString),
23    UTF16(BudgetedVec<u16>),
24}
25
26impl TermBuffer {
27    pub fn new(input: &TokenTerm, budget: &MemoryBudget) -> Self {
28        if input.as_str().is_some() {
29            Self::Unicode(BudgetedString::new(budget))
30        } else {
31            Self::UTF16(BudgetedVec::new(budget))
32        }
33    }
34
35    pub fn push(&mut self, character: Result<char, u16>) -> AnalysisResult<()> {
36        match (self, character) {
37            (Self::Unicode(text), Ok(character)) => text.push(character)?,
38            (Self::UTF16(units), Ok(character)) => {
39                for unit in character.encode_utf16(&mut [0; 2]) {
40                    units.push(*unit)?;
41                }
42            }
43            (Self::UTF16(units), Err(unit)) => units.push(unit)?,
44            (Self::Unicode(_), Err(_)) => unreachable!("scalar input preserves scalar output"),
45        }
46        Ok(())
47    }
48
49    pub fn finish(
50        self,
51        poll: &mut dyn FnMut() -> AnalysisResult<()>,
52    ) -> AnalysisResult<Budgeted<TokenTerm>> {
53        poll()?;
54        match self {
55            Self::Unicode(text) => {
56                let (text, memory) = text.into_parts();
57                Ok(Budgeted::new(TokenTerm::from(text), memory))
58            }
59            Self::UTF16(units) => {
60                let (units, memory) = units.into_parts();
61                TokenTerm::from_utf16_budgeted(Budgeted::new(units, memory), poll)
62            }
63        }
64    }
65}
66
67impl TokenTerm {
68    pub(crate) fn allocation_bytes(&self) -> usize {
69        match &self.0 {
70            Representation::Unicode(text) => text.capacity(),
71            Representation::UTF16(units) => units.capacity() * size_of::<u16>(),
72        }
73    }
74
75    pub(crate) fn character_count_with_control(
76        &self,
77        poll: &mut dyn FnMut() -> AnalysisResult<()>,
78    ) -> AnalysisResult<usize> {
79        poll()?;
80        let mut length = 0usize;
81        for (index, _) in self.characters().enumerate() {
82            if index % 1024 == 0 {
83                poll()?;
84            }
85            length += 1;
86        }
87        Ok(length)
88    }
89
90    /// Copy this term into independently reserved storage without changing scalar or raw-unit identity.
91    pub fn clone_budgeted(
92        &self,
93        budget: &MemoryBudget,
94        mut poll: impl FnMut() -> AnalysisResult<()>,
95    ) -> AnalysisResult<Budgeted<Self>> {
96        match &self.0 {
97            Representation::Unicode(text) => {
98                let (text, memory) =
99                    crate::allocation::copy_text(text, budget, &mut poll)?.into_parts();
100                Ok(Budgeted::new(Self::from(text), memory))
101            }
102            Representation::UTF16(units) => {
103                let (units, memory) =
104                    crate::allocation::copy_units(units, budget, &mut poll)?.into_parts();
105                Ok(Budgeted::new(Self(Representation::UTF16(units)), memory))
106            }
107        }
108    }
109
110    pub(crate) fn boundaries_budgeted(
111        &self,
112        budget: &MemoryBudget,
113        poll: &mut dyn FnMut() -> AnalysisResult<()>,
114    ) -> AnalysisResult<Budgeted<Vec<TermBoundary>>> {
115        poll()?;
116        let length = self
117            .character_count_with_control(poll)?
118            .checked_add(1)
119            .ok_or(MemoryError::SizeOverflow)?;
120        let scalar = self.as_str().is_some();
121        let mut output = BudgetedVec::new(budget);
122        output.reserve(length)?;
123        let mut boundary = TermBoundary {
124            offset: 0,
125            utf16: 0,
126        };
127        output.push(boundary)?;
128        for (index, character) in self.characters().enumerate() {
129            if index % 1024 == 0 {
130                poll()?;
131            }
132            let utf16 = character.map_or(1, char::len_utf16);
133            boundary.utf16 += utf16;
134            boundary.offset += if scalar {
135                character.expect("scalar representation").len_utf8()
136            } else {
137                utf16
138            };
139            output.push(boundary)?;
140        }
141        let (output, memory) = output.into_parts();
142        Ok(Budgeted::new(output, memory))
143    }
144
145    pub(crate) fn substring_budgeted(
146        &self,
147        range: std::ops::Range<usize>,
148        budget: &MemoryBudget,
149        poll: &mut dyn FnMut() -> AnalysisResult<()>,
150    ) -> AnalysisResult<Budgeted<Self>> {
151        match &self.0 {
152            Representation::Unicode(text) => {
153                let (text, memory) =
154                    crate::allocation::copy_text(&text[range], budget, poll)?.into_parts();
155                Ok(Budgeted::new(Self::from(text), memory))
156            }
157            Representation::UTF16(units) => Self::from_utf16_budgeted(
158                crate::allocation::copy_units(&units[range], budget, poll)?,
159                poll,
160            ),
161        }
162    }
163
164    pub(crate) fn eq_with_control(
165        &self,
166        other: &Self,
167        poll: &mut dyn FnMut() -> AnalysisResult<()>,
168    ) -> AnalysisResult<bool> {
169        fn compare<T: PartialEq>(
170            left: &[T],
171            right: &[T],
172            poll: &mut dyn FnMut() -> AnalysisResult<()>,
173        ) -> AnalysisResult<bool> {
174            poll()?;
175            if left.len() != right.len() {
176                return Ok(false);
177            }
178            for (left, right) in left.chunks(1024).zip(right.chunks(1024)) {
179                poll()?;
180                if left != right {
181                    return Ok(false);
182                }
183            }
184            Ok(true)
185        }
186        match (&self.0, &other.0) {
187            (Representation::Unicode(left), Representation::Unicode(right)) => {
188                compare(left.as_bytes(), right.as_bytes(), poll)
189            }
190            (Representation::UTF16(left), Representation::UTF16(right)) => {
191                compare(left, right, poll)
192            }
193            _ => {
194                poll()?;
195                Ok(false)
196            }
197        }
198    }
199
200    /// A representation order for internal lookup, with canonical term equality.
201    pub(crate) fn cmp_with_control(
202        &self,
203        other: &Self,
204        poll: &mut dyn FnMut() -> AnalysisResult<()>,
205    ) -> AnalysisResult<std::cmp::Ordering> {
206        fn compare<T: Ord>(
207            left: &[T],
208            right: &[T],
209            poll: &mut dyn FnMut() -> AnalysisResult<()>,
210        ) -> AnalysisResult<std::cmp::Ordering> {
211            poll()?;
212            for (left, right) in left.chunks(1024).zip(right.chunks(1024)) {
213                poll()?;
214                let order = left.cmp(right);
215                if !order.is_eq() {
216                    return Ok(order);
217                }
218            }
219            Ok(left.len().cmp(&right.len()))
220        }
221        match (&self.0, &other.0) {
222            (Representation::Unicode(left), Representation::Unicode(right)) => {
223                compare(left.as_bytes(), right.as_bytes(), poll)
224            }
225            (Representation::UTF16(left), Representation::UTF16(right)) => {
226                compare(left, right, poll)
227            }
228            (Representation::Unicode(_), Representation::UTF16(_)) => {
229                poll()?;
230                Ok(std::cmp::Ordering::Less)
231            }
232            (Representation::UTF16(_), Representation::Unicode(_)) => {
233                poll()?;
234                Ok(std::cmp::Ordering::Greater)
235            }
236        }
237    }
238
239    /// Consume reserved UTF-16 units, preserving isolated surrogates without replacement.
240    ///
241    /// Valid input reserves the exact UTF-8 buffer before decoding and releases the UTF-16 lease only after freeing its buffer. The input must carry the reservation for its capacity. Cancellation is checked during both validation and conversion.
242    pub fn from_utf16_budgeted(
243        input: Budgeted<Vec<u16>>,
244        mut poll: impl FnMut() -> AnalysisResult<()>,
245    ) -> AnalysisResult<Budgeted<Self>> {
246        poll()?;
247        let mut length = 0usize;
248        for (index, character) in char::decode_utf16(input.iter().copied()).enumerate() {
249            if index % 1024 == 0 {
250                poll()?;
251            }
252            let Ok(character) = character else {
253                let (units, memory) = input.into_parts();
254                return Ok(Budgeted::new(Self(Representation::UTF16(units)), memory));
255            };
256            length = length
257                .checked_add(character.len_utf8())
258                .ok_or(MemoryError::SizeOverflow)?;
259        }
260        let (units, memory) = input.into_parts();
261        let budget = memory.budget().clone();
262        let input = Budgeted::new(units, memory);
263        let mut output = BudgetedString::new(&budget);
264        output.reserve(length)?;
265        for (index, character) in char::decode_utf16(input.iter().copied()).enumerate() {
266            if index % 1024 == 0 {
267                poll()?;
268            }
269            output.push(character.expect("validated UTF-16"))?;
270        }
271        poll()?;
272        drop(input);
273        let (text, memory) = output.into_parts();
274        Ok(Budgeted::new(Self::from(text), memory))
275    }
276}
277
278#[cfg(test)]
279mod tests;
uqa_analysis/term/allocation.rs

uqa_analysis/term/
allocation.rs
