uqa_storage/
term_key.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Canonical binary posting keys for scalar strings and unpaired UTF-16 terms.
8
9use uqa_analysis::TokenTerm;
10
11mod allocation;
12
13use crate::{StorageBackendError, StorageBackendResult};
14
15/// A tagged term key: zero plus UTF-8, or one plus big-endian UTF-16 containing unpaired units.
16///
17/// Scalar text has exactly one representation, including supplementary characters and the empty string. Byte ordering is the persistent vocabulary order; it is not linguistic collation. Field/table names remain separate key components.
18#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
19pub struct TokenTermKey(Vec<u8>);
20
21impl TokenTermKey {
22    pub fn from_text(text: &str) -> Self {
23        let mut bytes = Vec::with_capacity(text.len() + 1);
24        bytes.push(0);
25        bytes.extend_from_slice(text.as_bytes());
26        Self(bytes)
27    }
28
29    pub fn from_term(term: &TokenTerm) -> Self {
30        if let Some(text) = term.as_str() {
31            return Self::from_text(text);
32        }
33        let units = term.utf16();
34        let mut bytes = Vec::with_capacity(units.len() * 2 + 1);
35        bytes.push(1);
36        for unit in units.iter() {
37            bytes.extend_from_slice(&unit.to_be_bytes());
38        }
39        Self(bytes)
40    }
41
42    pub fn from_bytes(bytes: Vec<u8>) -> StorageBackendResult<Self> {
43        Self::validate(&bytes)?;
44        Ok(Self(bytes))
45    }
46
47    /// Borrow scalar text when this key contains a valid Unicode string.
48    pub fn as_str(&self) -> Option<&str> {
49        (self.0[0] == 0).then(|| std::str::from_utf8(&self.0[1..]).expect("validated UTF-8 key"))
50    }
51
52    pub fn as_bytes(&self) -> &[u8] {
53        &self.0
54    }
55
56    pub(crate) fn allocated_bytes(&self) -> usize {
57        self.0.capacity()
58    }
59
60    pub fn into_bytes(self) -> Vec<u8> {
61        self.0
62    }
63
64    pub fn to_term(&self) -> TokenTerm {
65        match self.0[0] {
66            0 => TokenTerm::from(std::str::from_utf8(&self.0[1..]).expect("validated UTF-8 key")),
67            1 => TokenTerm::from_utf16(
68                self.0[1..]
69                    .chunks_exact(2)
70                    .map(|pair| u16::from_be_bytes([pair[0], pair[1]]))
71                    .collect(),
72            ),
73            _ => unreachable!("validated token key tag"),
74        }
75    }
76
77    fn validate(bytes: &[u8]) -> StorageBackendResult<()> {
78        let invalid = || StorageBackendError::Other("invalid canonical token term key".into());
79        match bytes.split_first() {
80            Some((0, bytes)) => std::str::from_utf8(bytes)
81                .map(|_| ())
82                .map_err(|_| invalid()),
83            Some((1, bytes)) if bytes.len() % 2 == 0 => {
84                let units = bytes
85                    .chunks_exact(2)
86                    .map(|pair| u16::from_be_bytes([pair[0], pair[1]]));
87                if !char::decode_utf16(units).any(|unit| unit.is_err()) {
88                    return Err(invalid());
89                }
90                Ok(())
91            }
92            _ => Err(invalid()),
93        }
94    }
95}
96
97impl From<String> for TokenTermKey {
98    fn from(text: String) -> Self {
99        Self::from_text(&text)
100    }
101}
102
103impl From<&str> for TokenTermKey {
104    fn from(text: &str) -> Self {
105        Self::from_text(text)
106    }
107}
uqa_storage/term_key.rs

uqa_storage/
term_key.rs
