cadmpeg_step/
lex.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Byte-oriented ISO 10303-21 lexical analysis.
3
4use std::ops::Range;
5
6/// A lexical token with its exact source-byte extent.
7#[derive(Debug, Clone, PartialEq)]
8pub struct Token {
9    /// Parsed token category.
10    pub kind: TokenKind,
11    /// Half-open byte range in the exchange structure.
12    pub span: Range<usize>,
13}
14
15/// Part 21 token categories.
16#[derive(Debug, Clone, PartialEq)]
17pub enum TokenKind {
18    /// Standard keyword or entity name.
19    Name(String),
20    /// User-defined `!`-prefixed keyword.
21    UserName(String),
22    /// Numeric `#`-prefixed entity-instance name.
23    Instance(u64),
24    /// Signed decimal integer.
25    Integer(i64),
26    /// Decimal real, including an optional exponent.
27    Real(f64),
28    /// Dot-delimited enumeration or logical literal.
29    Enumeration(String),
30    /// Bytes between apostrophe delimiters, before escape decoding.
31    String(Vec<u8>),
32    /// Decoded quoted hexadecimal binary literal.
33    Binary(BinaryValue),
34    /// Edition-3 resource token.
35    Resource(String),
36    /// Opening parenthesis.
37    LParen,
38    /// Closing parenthesis.
39    RParen,
40    /// Parameter separator.
41    Comma,
42    /// Statement terminator.
43    Semicolon,
44    /// Assignment operator.
45    Equals,
46    /// Omitted-value marker `$`.
47    Omitted,
48    /// Derived-value marker `*`.
49    Derived,
50}
51
52/// Binary literal payload packed most-significant nibble first.
53#[derive(Debug, Clone, PartialEq, Eq)]
54pub struct BinaryValue {
55    /// Number of significant payload bits.
56    pub bit_len: usize,
57    /// Packed bytes; unused low-order bits in the final byte are zero.
58    pub data: Vec<u8>,
59}
60
61/// Lexical failure with a stable byte position.
62#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
63#[error("{message} at byte {offset}")]
64pub struct LexError {
65    /// Byte offset at which tokenization failed.
66    pub offset: usize,
67    /// Violated lexical invariant.
68    pub message: String,
69}
70
71/// Tokenize one complete clear-text exchange structure.
72pub fn lex(input: &[u8]) -> Result<Vec<Token>, LexError> {
73    let mut lexer = Lexer { input, at: 0 };
74    let mut tokens = Vec::new();
75    while lexer.skip_trivia()? {
76        tokens.push(lexer.token()?);
77        if matches!(
78            tokens.last().map(|token| &token.kind),
79            Some(TokenKind::Semicolon)
80        ) && matches!(
81            tokens.get(tokens.len().saturating_sub(2)).map(|token| &token.kind),
82            Some(TokenKind::Name(name)) if name == "SIGNATURE"
83        ) {
84            lexer.skip_signature_payload()?;
85        }
86    }
87    Ok(tokens)
88}
89
90struct Lexer<'a> {
91    input: &'a [u8],
92    at: usize,
93}
94
95impl Lexer<'_> {
96    fn skip_signature_payload(&mut self) -> Result<(), LexError> {
97        let start = self.at;
98        let tail = &self.input[start..];
99        let exchange_end = tail
100            .windows(b"END-ISO-10303-21".len())
101            .position(|window| window == b"END-ISO-10303-21")
102            .ok_or_else(|| Self::error(start, "unterminated signature section"))?;
103        let section_end = tail[..exchange_end]
104            .windows(b"ENDSEC".len())
105            .rposition(|window| window == b"ENDSEC")
106            .ok_or_else(|| Self::error(start, "unterminated signature section"))?;
107        self.at = start + section_end;
108        Ok(())
109    }
110
111    fn skip_trivia(&mut self) -> Result<bool, LexError> {
112        loop {
113            while self.input.get(self.at).is_some_and(u8::is_ascii_whitespace) {
114                self.at += 1;
115            }
116            if self.input.get(self.at..self.at + 2) != Some(b"/*") {
117                return Ok(self.at < self.input.len());
118            }
119            let start = self.at;
120            self.at += 2;
121            let Some(end) = self.input[self.at..].windows(2).position(|w| w == b"*/") else {
122                return Err(Self::error(start, "unterminated comment"));
123            };
124            self.at += end + 2;
125        }
126    }
127
128    fn token(&mut self) -> Result<Token, LexError> {
129        let start = self.at;
130        let byte = self.input[self.at];
131        let kind = match byte {
132            b'(' => self.one(TokenKind::LParen),
133            b')' => self.one(TokenKind::RParen),
134            b',' => self.one(TokenKind::Comma),
135            b';' => self.one(TokenKind::Semicolon),
136            b'=' => self.one(TokenKind::Equals),
137            b'$' => self.one(TokenKind::Omitted),
138            b'*' => self.one(TokenKind::Derived),
139            b'#' => self.instance()?,
140            b'\'' => self.string()?,
141            b'"' => self.binary()?,
142            b'<' => self.resource()?,
143            b'.' if self
144                .input
145                .get(self.at + 1)
146                .is_some_and(u8::is_ascii_alphabetic) =>
147            {
148                self.enumeration()?
149            }
150            b'!' => self.user_name()?,
151            b'+' | b'-' | b'0'..=b'9' | b'.' => self.number()?,
152            b if b.is_ascii_alphabetic() => self.name(),
153            _ => return Err(Self::error(start, "unexpected byte")),
154        };
155        Ok(Token {
156            kind,
157            span: start..self.at,
158        })
159    }
160
161    fn one(&mut self, kind: TokenKind) -> TokenKind {
162        self.at += 1;
163        kind
164    }
165
166    fn name(&mut self) -> TokenKind {
167        let start = self.at;
168        self.at += 1;
169        while self
170            .input
171            .get(self.at)
172            .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_' || *b == b'-')
173        {
174            self.at += 1;
175        }
176        TokenKind::Name(String::from_utf8_lossy(&self.input[start..self.at]).to_ascii_uppercase())
177    }
178
179    fn user_name(&mut self) -> Result<TokenKind, LexError> {
180        let start = self.at;
181        self.at += 1;
182        if !self.input.get(self.at).is_some_and(u8::is_ascii_alphabetic) {
183            return Err(Self::error(start, "user-defined name has no identifier"));
184        }
185        let TokenKind::Name(name) = self.name() else {
186            unreachable!()
187        };
188        Ok(TokenKind::UserName(name))
189    }
190
191    fn instance(&mut self) -> Result<TokenKind, LexError> {
192        let start = self.at;
193        self.at += 1;
194        let digits = self.at;
195        while self.input.get(self.at).is_some_and(u8::is_ascii_digit) {
196            self.at += 1;
197        }
198        if digits == self.at {
199            return Err(Self::error(start, "instance name has no digits"));
200        }
201        let value = std::str::from_utf8(&self.input[digits..self.at])
202            .ok()
203            .and_then(|s| s.parse().ok())
204            .ok_or_else(|| Self::error(start, "instance name is out of range"))?;
205        Ok(TokenKind::Instance(value))
206    }
207
208    fn number(&mut self) -> Result<TokenKind, LexError> {
209        let start = self.at;
210        if matches!(self.input[self.at], b'+' | b'-') {
211            self.at += 1;
212        }
213        let mut dot = false;
214        let mut exponent = false;
215        while let Some(&b) = self.input.get(self.at) {
216            match b {
217                b'0'..=b'9' => self.at += 1,
218                b'.' if !dot && !exponent => {
219                    dot = true;
220                    self.at += 1;
221                }
222                b'E' | b'e' | b'D' | b'd' if !exponent => {
223                    exponent = true;
224                    self.at += 1;
225                    if self
226                        .input
227                        .get(self.at)
228                        .is_some_and(|b| matches!(b, b'+' | b'-'))
229                    {
230                        self.at += 1;
231                    }
232                }
233                _ => break,
234            }
235        }
236        let raw = std::str::from_utf8(&self.input[start..self.at]).unwrap_or_default();
237        if dot || exponent {
238            let parsed = if raw
239                .as_bytes()
240                .iter()
241                .any(|byte| matches!(byte, b'D' | b'd'))
242            {
243                raw.replace(['D', 'd'], "E").parse()
244            } else {
245                raw.parse()
246            };
247            parsed
248                .map(TokenKind::Real)
249                .map_err(|_| Self::error(start, "invalid real"))
250        } else {
251            raw.parse()
252                .map(TokenKind::Integer)
253                .map_err(|_| Self::error(start, "invalid integer"))
254        }
255    }
256
257    fn enumeration(&mut self) -> Result<TokenKind, LexError> {
258        let start = self.at;
259        self.at += 1;
260        let name_start = self.at;
261        while self
262            .input
263            .get(self.at)
264            .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_')
265        {
266            self.at += 1;
267        }
268        if self.input.get(self.at) != Some(&b'.') {
269            return Err(Self::error(start, "unterminated enumeration"));
270        }
271        let name = String::from_utf8_lossy(&self.input[name_start..self.at]).to_ascii_uppercase();
272        self.at += 1;
273        Ok(TokenKind::Enumeration(name))
274    }
275
276    fn string(&mut self) -> Result<TokenKind, LexError> {
277        let start = self.at;
278        self.at += 1;
279        let mut bytes = Vec::new();
280        loop {
281            match self.input.get(self.at).copied() {
282                Some(b'\'') if self.input.get(self.at + 1) == Some(&b'\'') => {
283                    bytes.extend_from_slice(b"''");
284                    self.at += 2;
285                }
286                Some(b'\'') => {
287                    self.at += 1;
288                    return Ok(TokenKind::String(bytes));
289                }
290                Some(byte) => {
291                    bytes.push(byte);
292                    self.at += 1;
293                }
294                None => return Err(Self::error(start, "unterminated string")),
295            }
296        }
297    }
298
299    fn binary(&mut self) -> Result<TokenKind, LexError> {
300        let start = self.at;
301        self.at += 1;
302        let content = self.at;
303        while self.input.get(self.at).is_some_and(u8::is_ascii_hexdigit) {
304            self.at += 1;
305        }
306        if self.input.get(self.at) != Some(&b'"') {
307            return Err(Self::error(start, "invalid binary literal"));
308        }
309        let raw = &self.input[content..self.at];
310        let Some((&indicator, digits)) = raw.split_first() else {
311            return Err(Self::error(
312                start,
313                "binary literal has no unused-bit indicator",
314            ));
315        };
316        let unused_bits = match indicator {
317            b'0'..=b'3' => indicator - b'0',
318            _ => {
319                return Err(Self::error(
320                    start,
321                    "binary unused-bit indicator exceeds three",
322                ))
323            }
324        };
325        if digits.is_empty() && unused_bits != 0 {
326            return Err(Self::error(start, "empty binary payload has unused bits"));
327        }
328        let nibbles = digits
329            .iter()
330            .map(|byte| match byte {
331                b'0'..=b'9' => byte - b'0',
332                b'a'..=b'f' => byte - b'a' + 10,
333                b'A'..=b'F' => byte - b'A' + 10,
334                _ => unreachable!("binary digits were validated as ASCII hexadecimal"),
335            })
336            .collect::<Vec<_>>();
337        if unused_bits != 0
338            && nibbles
339                .last()
340                .is_some_and(|nibble| nibble & ((1 << unused_bits) - 1) != 0)
341        {
342            return Err(Self::error(start, "unused binary bits are not zero"));
343        }
344        let mut data = Vec::with_capacity(nibbles.len().div_ceil(2));
345        for chunk in nibbles.chunks(2) {
346            data.push((chunk[0] << 4) | chunk.get(1).copied().unwrap_or(0));
347        }
348        let bit_len = digits.len() * 4 - usize::from(unused_bits);
349        self.at += 1;
350        Ok(TokenKind::Binary(BinaryValue { bit_len, data }))
351    }
352
353    fn resource(&mut self) -> Result<TokenKind, LexError> {
354        let start = self.at;
355        self.at += 1;
356        let content = self.at;
357        while self.input.get(self.at).is_some_and(|byte| *byte != b'>') {
358            self.at += 1;
359        }
360        if self.input.get(self.at) != Some(&b'>') {
361            return Err(Self::error(start, "unterminated resource token"));
362        }
363        let value = String::from_utf8(self.input[content..self.at].to_vec())
364            .map_err(|_| Self::error(start, "resource token is not UTF-8"))?;
365        self.at += 1;
366        Ok(TokenKind::Resource(value))
367    }
368
369    fn error(offset: usize, message: &str) -> LexError {
370        LexError {
371            offset,
372            message: message.into(),
373        }
374    }
375}
cadmpeg_step/lex.rs

cadmpeg_step/
lex.rs
