ifc_lite_core/parser/
tokenizer.rs

1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
5//! nom-combinator tokenizer for STEP/IFC entity lines.
6//!
7//! Zero-copy tokenization: string-like tokens borrow their original bytes.
8
9use nom::{
10    branch::alt,
11    bytes::complete::{take_while, take_while1},
12    character::complete::{char, digit1, one_of},
13    combinator::{map, map_res, opt, recognize},
14    multi::separated_list0,
15    sequence::{delimited, pair, preceded, tuple},
16    IResult,
17};
18
19use crate::error::{Error, Result};
20use crate::generated::IfcType;
21
22/// STEP/IFC token.
23///
24/// String-like tokens borrow their original bytes. Decode them only at a
25/// user-facing boundary so malformed real-world encodings cannot invalidate
26/// the structural parser.
27#[derive(Debug, Clone, PartialEq)]
28pub enum Token<'a> {
29    /// Entity reference: #123
30    EntityRef(u32),
31    /// String literal: 'text'
32    String(&'a [u8]),
33    /// Integer: 42
34    Integer(i64),
35    /// Float: 3.14
36    Float(f64),
37    /// Enum: .TRUE., .FALSE., .UNKNOWN.
38    Enum(&'a [u8]),
39    /// List: (1, 2, 3)
40    List(Vec<Token<'a>>),
41    /// Typed value: IFCPARAMETERVALUE(0.), IFCBOOLEAN(.T.)
42    TypedValue(&'a [u8], Vec<Token<'a>>),
43    /// Null value: $
44    Null,
45    /// Asterisk (derived value): *
46    Derived,
47}
48
49/// Parse entity reference: #123
50fn entity_ref(input: &[u8]) -> IResult<&[u8], Token<'_>> {
51    map(
52        preceded(char('#'), map_res(digit1, lexical_core::parse::<u32>)),
53        Token::EntityRef,
54    )(input)
55}
56
57/// Parse string literal: 'text' or "text"
58/// IFC uses '' to escape a single quote within a string
59/// Uses memchr for SIMD-accelerated quote searching
60fn string_literal(input: &[u8]) -> IResult<&[u8], Token<'_>> {
61    // Helper to parse string content with escaped quotes - SIMD optimized
62    #[inline]
63    fn parse_string_content(input: &[u8], quote_byte: u8) -> IResult<&[u8], &[u8]> {
64        let bytes = input;
65        let mut pos = 0;
66
67        // Use memchr for SIMD-accelerated searching
68        while let Some(found) = memchr::memchr(quote_byte, &bytes[pos..]) {
69            let idx = pos + found;
70            // Check if it's an escaped quote (doubled)
71            if idx + 1 < bytes.len() && bytes[idx + 1] == quote_byte {
72                pos = idx + 2; // Skip escaped quote pair
73                continue;
74            }
75            // End of string found
76            return Ok((&input[idx..], &input[..idx]));
77        }
78
79        // No closing quote found
80        Err(nom::Err::Error(nom::error::Error::new(
81            input,
82            nom::error::ErrorKind::Char,
83        )))
84    }
85
86    alt((
87        map(
88            delimited(char('\''), |i| parse_string_content(i, b'\''), char('\'')),
89            Token::String,
90        ),
91        map(
92            delimited(char('"'), |i| parse_string_content(i, b'"'), char('"')),
93            Token::String,
94        ),
95    ))(input)
96}
97
98/// Parse integer: 42, -42
99/// Uses lexical-core for 10x faster parsing
100#[inline]
101fn integer(input: &[u8]) -> IResult<&[u8], Token<'_>> {
102    map_res(recognize(tuple((opt(char('-')), digit1))), |s: &[u8]| {
103        lexical_core::parse::<i64>(s)
104            .map(Token::Integer)
105            .map_err(|_| "parse error")
106    })(input)
107}
108
109/// Parse float: 3.14, -3.14, 1.5E-10, 0., 1.
110/// IFC allows floats like "0." without decimal digits
111/// Uses lexical-core for 10x faster parsing
112#[inline]
113fn float(input: &[u8]) -> IResult<&[u8], Token<'_>> {
114    map_res(
115        recognize(tuple((
116            opt(char('-')),
117            digit1,
118            char('.'),
119            opt(digit1), // Made optional to support "0." format
120            opt(tuple((one_of("eE"), opt(one_of("+-")), digit1))),
121        ))),
122        |s: &[u8]| {
123            lexical_core::parse::<f64>(s)
124                .map(Token::Float)
125                .map_err(|_| "parse error")
126        },
127    )(input)
128}
129
130/// Parse enum: .TRUE., .FALSE., .UNKNOWN., .ELEMENT.
131fn enum_value(input: &[u8]) -> IResult<&[u8], Token<'_>> {
132    map(
133        delimited(
134            char('.'),
135            take_while1(|c: u8| c.is_ascii_alphanumeric() || c == b'_'),
136            char('.'),
137        ),
138        Token::Enum,
139    )(input)
140}
141
142/// Parse null: $
143fn null(input: &[u8]) -> IResult<&[u8], Token<'_>> {
144    map(char('$'), |_| Token::Null)(input)
145}
146
147/// Parse derived: *
148fn derived(input: &[u8]) -> IResult<&[u8], Token<'_>> {
149    map(char('*'), |_| Token::Derived)(input)
150}
151
152/// Maximum nesting depth for token recursion (list and typed-value bodies).
153///
154/// Each `(` in the input bumps depth by one. Real-world IFC entities rarely
155/// nest beyond 5-10 levels; 256 leaves comfortable headroom while keeping
156/// the stack bounded against pathological inputs.
157const MAX_NESTING_DEPTH: u32 = 256;
158
159/// Parse typed value: IFCPARAMETERVALUE(0.), IFCBOOLEAN(.T.)
160fn typed_value_at_depth(input: &[u8], depth: u32) -> IResult<&[u8], Token<'_>> {
161    map(
162        pair(
163            // Type name (all caps with optional numbers/underscores)
164            take_while1(|c: u8| c.is_ascii_alphanumeric() || c == b'_'),
165            // Arguments
166            delimited(
167                char('('),
168                separated_list0(delimited(ws, char(','), ws), move |i| {
169                    token_at_depth(i, depth)
170                }),
171                char(')'),
172            ),
173        ),
174        |(type_name, args)| Token::TypedValue(type_name, args),
175    )(input)
176}
177
178/// Skip whitespace
179fn ws(input: &[u8]) -> IResult<&[u8], ()> {
180    map(take_while(|c: u8| c.is_ascii_whitespace()), |_| ())(input)
181}
182
183/// Parse a token with optional surrounding whitespace
184/// Optimized ordering: test cheapest patterns first (single-char markers)
185fn token(input: &[u8]) -> IResult<&[u8], Token<'_>> {
186    token_at_depth(input, 0)
187}
188
189fn token_at_depth(input: &[u8], depth: u32) -> IResult<&[u8], Token<'_>> {
190    if depth > MAX_NESTING_DEPTH {
191        return Err(nom::Err::Failure(nom::error::Error::new(
192            input,
193            nom::error::ErrorKind::TooLarge,
194        )));
195    }
196    delimited(
197        ws,
198        alt((
199            // Single-char markers first (O(1) check)
200            null,       // $
201            derived,    // *
202            entity_ref, // # + digits
203            // Then by complexity
204            enum_value,     // .XXX.
205            string_literal, // 'xxx'
206            move |i| list_at_depth(i, depth + 1), // (...)
207            // Numbers: float before integer since float includes '.'
208            float,
209            integer,
210            // IFCPARAMETERVALUE(0.) - most expensive, last
211            move |i| typed_value_at_depth(i, depth + 1),
212        )),
213        ws,
214    )(input)
215}
216
217/// Parse list: (1, 2, 3) or nested lists
218/// Test-only wrapper for the depth-0 entry into list.
219#[cfg(test)]
220fn list(input: &[u8]) -> IResult<&[u8], Token<'_>> {
221    list_at_depth(input, 0)
222}
223
224fn list_at_depth(input: &[u8], depth: u32) -> IResult<&[u8], Token<'_>> {
225    map(
226        delimited(
227            char('('),
228            separated_list0(delimited(ws, char(','), ws), move |i| {
229                token_at_depth(i, depth)
230            }),
231            char(')'),
232        ),
233        Token::List,
234    )(input)
235}
236
237/// Parse a complete entity line from raw IFC bytes.
238/// Example: #123=IFCWALL('guid','owner',$,$,'name',$,$,$);
239// The nom `IResult` parser tuple type is intentionally explicit here; factoring
240// it into a `type` alias would obscure the parser combinator structure.
241#[allow(clippy::type_complexity)]
242pub fn parse_entity<'a, T>(input: &'a T) -> Result<(u32, IfcType, Vec<Token<'a>>)>
243where
244    T: AsRef<[u8]> + ?Sized,
245{
246    let input = input.as_ref();
247    let result: IResult<&[u8], (u32, &[u8], Vec<Token>)> = tuple((
248        // Entity ID: #123
249        delimited(
250            ws,
251            preceded(char('#'), map_res(digit1, lexical_core::parse::<u32>)),
252            ws,
253        ),
254        // Equals sign
255        preceded(
256            char('='),
257            // Entity type: IFCWALL
258            delimited(
259                ws,
260                take_while1(|c: u8| c.is_ascii_alphanumeric() || c == b'_'),
261                ws,
262            ),
263        ),
264        // Arguments: ('guid', 'owner', ...)
265        delimited(
266            char('('),
267            separated_list0(delimited(ws, char(','), ws), token),
268            tuple((char(')'), ws, char(';'))),
269        ),
270    ))(input);
271
272    match result {
273        Ok((_, (id, type_str, args))) => {
274            let type_str = std::str::from_utf8(type_str)
275                .map_err(|_| Error::parse(0, "Entity type is not ASCII/UTF-8"))?;
276            let ifc_type = IfcType::from_str(type_str);
277            Ok((id, ifc_type, args))
278        }
279        Err(e) => Err(Error::parse(0, format!("Failed to parse entity: {}", e))),
280    }
281}
282
283#[cfg(test)]
284mod tests {
285    use super::*;
286
287    /// Table-driven basic token parsing: (parser, input, expected token).
288    #[test]
289    #[allow(clippy::approx_constant)]
290    fn test_basic_tokens() {
291        type Parser = for<'a> fn(&'a [u8]) -> IResult<&'a [u8], Token<'a>>;
292        let cases: &[(Parser, &[u8], Token)] = &[
293            (entity_ref, b"#123", Token::EntityRef(123)),
294            (entity_ref, b"#0", Token::EntityRef(0)),
295            (string_literal, b"'hello'", Token::String(b"hello")),
296            (
297                string_literal,
298                b"'with spaces'",
299                Token::String(b"with spaces"),
300            ),
301            (integer, b"42", Token::Integer(42)),
302            (integer, b"-42", Token::Integer(-42)),
303            (integer, b"0", Token::Integer(0)),
304            (float, b"3.14", Token::Float(3.14)),
305            (float, b"-3.14", Token::Float(-3.14)),
306            (float, b"1.5E-10", Token::Float(1.5e-10)),
307            (enum_value, b".TRUE.", Token::Enum(b"TRUE")),
308            (enum_value, b".FALSE.", Token::Enum(b"FALSE")),
309            (enum_value, b".ELEMENT.", Token::Enum(b"ELEMENT")),
310        ];
311        for (parse, input, expected) in cases {
312            assert_eq!(
313                parse(input),
314                Ok((&b""[..], expected.clone())),
315                "tokenizing {input:?}"
316            );
317        }
318    }
319
320    #[test]
321    fn test_list() {
322        let result = list(b"(1,2,3)");
323        assert!(result.is_ok());
324        let (_, token) = result.unwrap();
325        match token {
326            Token::List(items) => {
327                assert_eq!(items.len(), 3);
328                assert_eq!(items[0], Token::Integer(1));
329                assert_eq!(items[1], Token::Integer(2));
330                assert_eq!(items[2], Token::Integer(3));
331            }
332            _ => panic!("Expected List token"),
333        }
334    }
335
336    #[test]
337    fn test_nested_list() {
338        let result = list(b"(1,(2,3),4)");
339        assert!(result.is_ok());
340        let (_, token) = result.unwrap();
341        match token {
342            Token::List(items) => {
343                assert_eq!(items.len(), 3);
344                assert_eq!(items[0], Token::Integer(1));
345                match &items[1] {
346                    Token::List(inner) => {
347                        assert_eq!(inner.len(), 2);
348                        assert_eq!(inner[0], Token::Integer(2));
349                        assert_eq!(inner[1], Token::Integer(3));
350                    }
351                    _ => panic!("Expected nested List"),
352                }
353                assert_eq!(items[2], Token::Integer(4));
354            }
355            _ => panic!("Expected List token"),
356        }
357    }
358
359    #[test]
360    fn test_parse_entity() {
361        let input = "#123=IFCWALL('guid','owner',$,$,'name',$,$,$);";
362        let result = parse_entity(input);
363        assert!(result.is_ok());
364        let (id, ifc_type, args) = result.unwrap();
365        assert_eq!(id, 123);
366        assert_eq!(ifc_type, IfcType::IfcWall);
367        assert_eq!(args.len(), 8);
368    }
369
370    #[test]
371    fn test_parse_entity_with_nested_list() {
372        // First test: simple list (should work)
373        let simple = "(0.,0.,1.)";
374        println!("Testing simple list: {}", simple);
375        let simple_result = list(simple.as_bytes());
376        println!("Simple list result: {:?}", simple_result);
377
378        // Second test: nested in entity (what's failing)
379        let input = "#9=IFCDIRECTION((0.,0.,1.));";
380        println!("\nTesting full entity: {}", input);
381        let result = parse_entity(input);
382
383        if let Err(ref e) = result {
384            println!("Parse error: {:?}", e);
385
386            // Try parsing just the arguments part
387            println!("\nTrying to parse just arguments: ((0.,0.,1.))");
388            let args_input = "((0.,0.,1.))";
389            let args_result = list(args_input.as_bytes());
390            println!("Args list result: {:?}", args_result);
391        }
392
393        assert!(result.is_ok(), "Failed to parse: {:?}", result);
394        let (id, _ifc_type, args) = result.unwrap();
395        assert_eq!(id, 9);
396        assert_eq!(args.len(), 1);
397        // First arg should be a list containing 3 floats
398        if let Token::List(inner) = &args[0] {
399            assert_eq!(inner.len(), 3);
400        } else {
401            panic!("Expected Token::List, got {:?}", args[0]);
402        }
403    }
404
405    /// Deeply nested list arguments must return an error rather than
406    /// recursing through the stack until it overflows.
407    #[test]
408    fn test_parse_entity_rejects_excessive_nesting() {
409        let n = (MAX_NESTING_DEPTH as usize) + 64;
410        let mut s = String::from("#1=IFCWALL(");
411        for _ in 0..n {
412            s.push('(');
413        }
414        s.push('1');
415        for _ in 0..n {
416            s.push(')');
417        }
418        s.push_str(");");
419        // Must not panic / overflow; must return Err.
420        assert!(parse_entity(&s).is_err());
421    }
422
423    /// Moderate nesting still parses successfully.
424    #[test]
425    fn test_parse_entity_accepts_moderate_nesting() {
426        let n = 32;
427        let mut s = String::from("#1=IFCWALL(");
428        for _ in 0..n {
429            s.push('(');
430        }
431        s.push('1');
432        for _ in 0..n {
433            s.push(')');
434        }
435        s.push_str(");");
436        assert!(parse_entity(&s).is_ok());
437    }
438
439    fn nested(n: usize) -> String {
440        let mut s = String::from("#1=IFCWALL(");
441        for _ in 0..n {
442            s.push('(');
443        }
444        s.push('1');
445        for _ in 0..n {
446            s.push(')');
447        }
448        s.push_str(");");
449        s
450    }
451
452    /// Boundary: parsing succeeds exactly at MAX_NESTING_DEPTH.
453    #[test]
454    fn test_parse_entity_accepts_exactly_max_nesting() {
455        assert!(parse_entity(&nested(MAX_NESTING_DEPTH as usize)).is_ok());
456    }
457
458    /// Boundary: parsing fails at MAX_NESTING_DEPTH + 1.
459    #[test]
460    fn test_parse_entity_rejects_one_over_max_nesting() {
461        assert!(parse_entity(&nested(MAX_NESTING_DEPTH as usize + 1)).is_err());
462    }
463}
ifc_lite_core/parser/tokenizer.rs

ifc_lite_core/parser/
tokenizer.rs
