lxdb_engine/
token.rs

1use std::str;
2
3use lxdb_core::ids::TokenId;
4use lxdb_format::TokenRecord;
5
6use crate::{EngineError, TokenRecordIter};
7
8/// A token resolved directly against the binary string table.
9///
10/// The text borrows from the underlying `BinaryDataset`, so no `String`
11/// allocation is required.
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub struct BinaryToken<'a> {
14    id: TokenId,
15    text: &'a str,
16}
17
18impl<'a> BinaryToken<'a> {
19    pub const fn new(id: TokenId, text: &'a str) -> Self {
20        Self { id, text }
21    }
22
23    pub const fn id(&self) -> TokenId {
24        self.id
25    }
26
27    pub const fn text(&self) -> &'a str {
28        self.text
29    }
30}
31
32/// Resolves token records lazily against the dataset string table.
33#[derive(Debug, Clone)]
34pub struct BinaryTokenIter<'a> {
35    records: TokenRecordIter<'a>,
36    string_table: &'a [u8],
37}
38
39impl<'a> BinaryTokenIter<'a> {
40    pub(crate) const fn new(records: TokenRecordIter<'a>, string_table: &'a [u8]) -> Self {
41        Self { records, string_table }
42    }
43
44    fn resolve(&self, record: TokenRecord) -> Result<BinaryToken<'a>, EngineError> {
45        let offset =
46            usize::try_from(record.offset()).map_err(|_| EngineError::TokenStringOutOfBounds {
47                token_id: record.id(),
48                offset: record.offset(),
49                length: record.length(),
50                table_length: self.string_table.len(),
51            })?;
52
53        let length = usize::try_from(record.length()).expect("u32 token length must fit in usize");
54
55        let end = offset.checked_add(length).ok_or(EngineError::TokenStringOutOfBounds {
56            token_id: record.id(),
57            offset: record.offset(),
58            length: record.length(),
59            table_length: self.string_table.len(),
60        })?;
61
62        let bytes =
63            self.string_table.get(offset..end).ok_or(EngineError::TokenStringOutOfBounds {
64                token_id: record.id(),
65                offset: record.offset(),
66                length: record.length(),
67                table_length: self.string_table.len(),
68            })?;
69
70        let text = str::from_utf8(bytes)
71            .map_err(|source| EngineError::InvalidTokenUtf8 { token_id: record.id(), source })?;
72
73        Ok(BinaryToken::new(TokenId::new(record.id()), text))
74    }
75}
76
77impl<'a> Iterator for BinaryTokenIter<'a> {
78    type Item = Result<BinaryToken<'a>, EngineError>;
79
80    fn next(&mut self) -> Option<Self::Item> {
81        let record = match self.records.next()? {
82            Ok(record) => record,
83            Err(error) => return Some(Err(error.into())),
84        };
85
86        Some(self.resolve(record))
87    }
88
89    fn size_hint(&self) -> (usize, Option<usize>) {
90        self.records.size_hint()
91    }
92}
93
94impl ExactSizeIterator for BinaryTokenIter<'_> {
95    fn len(&self) -> usize {
96        self.records.len()
97    }
98}
99
100impl std::iter::FusedIterator for BinaryTokenIter<'_> {}
101
102#[cfg(test)]
103mod tests {
104    use lxdb_format::TokenRecord;
105
106    use crate::{BinaryTokenIter, EngineError, RecordIter};
107
108    #[test]
109    fn resolves_tokens_without_allocating_strings() {
110        let first = TokenRecord::new(0, 0, 4);
111
112        let second = TokenRecord::new(1, 4, 8);
113
114        let mut records = Vec::new();
115
116        records.extend_from_slice(&first.encode());
117        records.extend_from_slice(&second.encode());
118
119        let string_table = b"rustlanguage";
120
121        let record_iter = RecordIter::<TokenRecord>::new(&records);
122
123        let mut tokens = BinaryTokenIter::new(record_iter, string_table);
124
125        assert_eq!(tokens.len(), 2);
126
127        let first =
128            tokens.next().expect("first token should exist").expect("first token should resolve");
129
130        assert_eq!(first.id().value(), 0);
131        assert_eq!(first.text(), "rust");
132
133        let second =
134            tokens.next().expect("second token should exist").expect("second token should resolve");
135
136        assert_eq!(second.id().value(), 1);
137        assert_eq!(second.text(), "language");
138
139        assert!(tokens.next().is_none());
140    }
141
142    #[test]
143    fn rejects_out_of_bounds_string_ranges() {
144        let record = TokenRecord::new(7, 10, 5);
145
146        let encoded = record.encode();
147
148        let record_iter = RecordIter::<TokenRecord>::new(&encoded);
149
150        let mut tokens = BinaryTokenIter::new(record_iter, b"short");
151
152        let error = tokens
153            .next()
154            .expect("token record should exist")
155            .expect_err("invalid string range should fail");
156
157        assert!(matches!(
158            error,
159            EngineError::TokenStringOutOfBounds {
160                token_id: 7,
161                offset: 10,
162                length: 5,
163                table_length: 5,
164            }
165        ));
166    }
167
168    #[test]
169    fn rejects_invalid_utf8_token_strings() {
170        let record = TokenRecord::new(3, 0, 2);
171
172        let encoded = record.encode();
173
174        let record_iter = RecordIter::<TokenRecord>::new(&encoded);
175
176        let invalid_utf8 = [0xFF, 0xFF];
177
178        let mut tokens = BinaryTokenIter::new(record_iter, &invalid_utf8);
179
180        let error = tokens
181            .next()
182            .expect("token record should exist")
183            .expect_err("invalid UTF-8 should fail");
184
185        assert!(matches!(error, EngineError::InvalidTokenUtf8 { token_id: 3, .. }));
186    }
187}
lxdb_engine/token.rs

lxdb_engine/
token.rs
