lxdb_format/
token_record.rs

1use crate::{BinaryRecord, FormatError};
2/// Number of bytes occupied by an encoded token record.
3pub const TOKEN_RECORD_SIZE: usize = 24;
4
5/// Binary representation of a token.
6///
7/// The text itself is stored separately inside the token string table.
8/// `offset` is relative to the beginning of that section.
9///
10/// Binary layout:
11///
12/// - token id: 4 bytes
13/// - reserved: 4 bytes
14/// - string offset: 8 bytes
15/// - string length: 4 bytes
16/// - reserved: 4 bytes
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub struct TokenRecord {
19    id: u32,
20    offset: u64,
21    length: u32,
22}
23
24impl TokenRecord {
25    pub const SIZE: usize = TOKEN_RECORD_SIZE;
26
27    pub const fn new(id: u32, offset: u64, length: u32) -> Self {
28        Self { id, offset, length }
29    }
30
31    pub const fn id(self) -> u32 {
32        self.id
33    }
34
35    pub const fn offset(self) -> u64 {
36        self.offset
37    }
38
39    pub const fn length(self) -> u32 {
40        self.length
41    }
42
43    pub const fn end(self) -> u64 {
44        self.offset + self.length as u64
45    }
46
47    pub fn encode(self) -> [u8; TOKEN_RECORD_SIZE] {
48        let mut bytes = [0_u8; TOKEN_RECORD_SIZE];
49
50        bytes[0..4].copy_from_slice(&self.id.to_le_bytes());
51
52        // bytes[4..8] are reserved and remain zero.
53
54        bytes[8..16].copy_from_slice(&self.offset.to_le_bytes());
55        bytes[16..20].copy_from_slice(&self.length.to_le_bytes());
56
57        // bytes[20..24] are reserved and remain zero.
58
59        bytes
60    }
61
62    pub fn decode(bytes: &[u8]) -> Result<Self, FormatError> {
63        if bytes.len() != Self::SIZE {
64            return Err(FormatError::UnexpectedRecordSize {
65                record: "token record",
66                expected: Self::SIZE,
67                found: bytes.len(),
68            });
69        }
70
71        let id =
72            u32::from_le_bytes(bytes[0..4].try_into().expect("token id must occupy four bytes"));
73
74        let offset = u64::from_le_bytes(
75            bytes[8..16].try_into().expect("token string offset must occupy eight bytes"),
76        );
77
78        let length = u32::from_le_bytes(
79            bytes[16..20].try_into().expect("token string length must occupy four bytes"),
80        );
81
82        Ok(Self::new(id, offset, length))
83    }
84}
85
86impl BinaryRecord for TokenRecord {
87    const SIZE: usize = TokenRecord::SIZE;
88
89    fn decode(bytes: &[u8]) -> Result<Self, FormatError> {
90        TokenRecord::decode(bytes)
91    }
92}
93
94#[cfg(test)]
95mod tests {
96    use super::{TOKEN_RECORD_SIZE, TokenRecord};
97
98    #[test]
99    fn encodes_token_record() {
100        let record = TokenRecord::new(42, 4_294_967_500, 128);
101
102        let bytes = record.encode();
103
104        assert_eq!(bytes.len(), TOKEN_RECORD_SIZE);
105
106        assert_eq!(
107            u32::from_le_bytes(bytes[0..4].try_into().expect("token id must occupy four bytes"),),
108            42,
109        );
110
111        assert_eq!(&bytes[4..8], &[0; 4]);
112
113        assert_eq!(
114            u64::from_le_bytes(
115                bytes[8..16].try_into().expect("string offset must occupy eight bytes"),
116            ),
117            4_294_967_500,
118        );
119
120        assert_eq!(
121            u32::from_le_bytes(
122                bytes[16..20].try_into().expect("string length must occupy four bytes"),
123            ),
124            128,
125        );
126
127        assert_eq!(&bytes[20..24], &[0; 4]);
128    }
129
130    #[test]
131    fn calculates_string_range_end() {
132        let record = TokenRecord::new(0, 500, 25);
133
134        assert_eq!(record.end(), 525);
135    }
136
137    #[test]
138    fn exposes_record_fields() {
139        let record = TokenRecord::new(7, 100, 12);
140
141        assert_eq!(record.id(), 7);
142        assert_eq!(record.offset(), 100);
143        assert_eq!(record.length(), 12);
144    }
145
146    #[test]
147    fn decodes_token_record() {
148        let original = TokenRecord::new(42, 8_589_934_592, 128);
149
150        let encoded = original.encode();
151
152        let decoded = TokenRecord::decode(&encoded).expect("encoded token record should decode");
153
154        assert_eq!(decoded, original);
155    }
156
157    #[test]
158    fn rejects_invalid_token_record_size() {
159        let bytes = [0_u8; TokenRecord::SIZE - 1];
160
161        let error = TokenRecord::decode(&bytes).expect_err("truncated token record should fail");
162
163        assert!(matches!(
164            error,
165            crate::FormatError::UnexpectedRecordSize {
166                record: "token record",
167                expected: TokenRecord::SIZE,
168                found,
169            } if found == TokenRecord::SIZE - 1
170        ));
171    }
172}
lxdb_format/token_record.rs

lxdb_format/
token_record.rs
