openbim_step/
parser.rs

1//! Semantic parser and event/sink interface.
2//!
3//! Parsing is structural: record names and parameters are retained without a
4//! domain schema. Unknown header and data records therefore survive a
5//! parse/write/reparse cycle.
6
7use crate::escape;
8use crate::lexer::{Lexer, Token};
9use crate::recovery::{Diagnostic, OnMalformed, ParseOptions, ParseOutcome};
10use crate::{
11    DataRecord, DataSection, Exchange, HeaderRecord, HeaderSection, InstanceId, Parameter, Record,
12    Span, Spanned, StepError,
13};
14
15/// A semantic parser event.
16#[derive(Debug, Clone, PartialEq)]
17pub enum Event<S = String> {
18    /// Entered `HEADER;`.
19    StartHeader,
20    /// Parsed one header record.
21    HeaderRecord(HeaderRecord<S>),
22    /// Reached the header `ENDSEC;`.
23    EndHeader,
24    /// Entered `DATA;`.
25    StartData,
26    /// Parsed one data record.
27    DataRecord(DataRecord<S>),
28    /// Reached the data `ENDSEC;`.
29    EndData,
30}
31
32/// Consumer for parse events.
33pub trait EventSink<S = String> {
34    /// Receives one event. Events are delivered in source order.
35    fn event(&mut self, event: Event<S>);
36}
37
38impl<S, F> EventSink<S> for F
39where
40    F: FnMut(Event<S>),
41{
42    fn event(&mut self, event: Event<S>) {
43        self(event);
44    }
45}
46
47/// Parses a physical file into an owned generic exchange structure.
48/// # Errors
49///
50/// Returns [`StepError`] for the wrong
51/// physical-file marker, or a spanned lexical/syntax diagnostic otherwise.
52pub fn parse(input: &[u8]) -> Result<Exchange, StepError> {
53    parse_with(input, ParseOptions::strict()).map(|outcome| outcome.exchange)
54}
55
56/// Parses a physical file under explicit [`ParseOptions`].
57///
58/// With [`OnMalformed::Skip`] an unparsable data record is reported as a
59/// [`Diagnostic`] and the parser resynchronizes on the next record or section
60/// end, so a consumer can load a damaged file and still show exactly what was
61/// dropped. Header structure stays strict under every policy: mandatory header
62/// records are file-level invariants, not recoverable payload.
63/// # Errors
64///
65/// Returns [`StepError`] for the wrong physical-file marker, for any header or
66/// section-structure defect, and for data defects when the policy is
67/// [`OnMalformed::Abort`].
68pub fn parse_with(input: &[u8], options: ParseOptions) -> Result<ParseOutcome, StepError> {
69    #[derive(Default)]
70    struct Builder {
71        header: HeaderSection,
72        data: DataSection,
73    }
74
75    impl EventSink for Builder {
76        fn event(&mut self, event: Event) {
77            match event {
78                Event::HeaderRecord(record) => self.header.records.push(record),
79                Event::DataRecord(record) => self.data.records.push(record),
80                Event::StartHeader | Event::EndHeader | Event::StartData | Event::EndData => {}
81            }
82        }
83    }
84
85    let mut builder = Builder::default();
86    let diagnostics = parse_events_with(input, &mut builder, options)?;
87    Ok(ParseOutcome {
88        exchange: Exchange {
89            header: builder.header,
90            data: builder.data,
91        },
92        diagnostics,
93    })
94}
95
96/// Parses a physical file and sends semantic records to `sink`.
97///
98/// Unlike [`parse`], this API does not accumulate an [`Exchange`]. It is useful
99/// for import pipelines that index, validate, or transform records as they are
100/// read.
101/// # Errors
102///
103/// Returns the same physical-file and syntax diagnostics as [`parse`].
104pub fn parse_events(input: &[u8], sink: &mut impl EventSink) -> Result<(), StepError> {
105    parse_events_with(input, sink, ParseOptions::strict()).map(|_| ())
106}
107
108/// Streams semantic records under explicit [`ParseOptions`], returning the
109/// non-fatal diagnostics collected on the way.
110/// # Errors
111///
112/// Returns the same physical-file, header, and structure diagnostics as
113/// [`parse_with`].
114pub fn parse_events_with(
115    input: &[u8],
116    sink: &mut impl EventSink,
117    options: ParseOptions,
118) -> Result<Vec<Diagnostic>, StepError> {
119    if !crate::is_step_file(input) {
120        return Err(StepError::not_step("missing ISO-10303-21 marker"));
121    }
122    let mut parser = Parser::new(input);
123    parser.options = options;
124    parser.parse(sink)?;
125    Ok(parser.diagnostics)
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq)]
129enum Phase {
130    BeforeStart,
131    BeforeHeader,
132    Header,
133    BeforeData,
134    Data,
135    BeforeEnd,
136    Done,
137}
138
139struct Parser<'a> {
140    input: &'a [u8],
141    lexer: Lexer<'a>,
142    lookahead: Option<Spanned<Token<'a>>>,
143    last_end: usize,
144    phase: Phase,
145    header_records_seen: usize,
146    options: ParseOptions,
147    diagnostics: Vec<Diagnostic>,
148}
149
150impl<'a> Parser<'a> {
151    fn new(input: &'a [u8]) -> Self {
152        Self {
153            input,
154            lexer: Lexer::new(input),
155            lookahead: None,
156            last_end: 0,
157            phase: Phase::BeforeStart,
158            header_records_seen: 0,
159            options: ParseOptions::strict(),
160            diagnostics: Vec::new(),
161        }
162    }
163
164    // Keeping section dispatch together makes the state-machine transitions auditable.
165    #[allow(clippy::too_many_lines)]
166    fn parse(&mut self, sink: &mut impl EventSink) -> Result<(), StepError> {
167        loop {
168            let token = match self.next() {
169                Ok(Some(token)) => token,
170                Ok(None) => break,
171                Err(error) => {
172                    // A lexical defect inside DATA is recoverable: the damaged
173                    // bytes belong to one record, not to the file structure.
174                    self.recover_or_fail(error, None)?;
175                    continue;
176                }
177            };
178            if self.phase == Phase::Done {
179                return Err(StepError::syntax(
180                    token.span,
181                    "content after END-ISO-10303-21",
182                ));
183            }
184            match token.value {
185                Token::Name(name) if name.eq_ignore_ascii_case(b"ISO-10303-21") => {
186                    if self.phase != Phase::BeforeStart {
187                        return Err(StepError::syntax(
188                            token.span,
189                            "unexpected ISO-10303-21 marker",
190                        ));
191                    }
192                    self.expect_semicolon("after ISO-10303-21")?;
193                    self.phase = Phase::BeforeHeader;
194                }
195                Token::Name(name) if name.eq_ignore_ascii_case(b"HEADER") => {
196                    if self.phase != Phase::BeforeHeader {
197                        return Err(StepError::syntax(token.span, "unexpected HEADER section"));
198                    }
199                    self.expect_semicolon("after HEADER")?;
200                    self.phase = Phase::Header;
201                    sink.event(Event::StartHeader);
202                }
203                Token::Name(name) if name.eq_ignore_ascii_case(b"DATA") => {
204                    if self.phase != Phase::BeforeData {
205                        return Err(StepError::syntax(token.span, "unexpected DATA section"));
206                    }
207                    self.expect_semicolon("after DATA")?;
208                    self.phase = Phase::Data;
209                    sink.event(Event::StartData);
210                }
211                Token::Name(name) if name.eq_ignore_ascii_case(b"ENDSEC") => {
212                    self.expect_semicolon("after ENDSEC")?;
213                    match self.phase {
214                        Phase::Header => {
215                            if self.header_records_seen < 3 {
216                                return Err(StepError::syntax(
217                                    token.span,
218                                    "missing mandatory STEP header record",
219                                ));
220                            }
221                            sink.event(Event::EndHeader);
222                            self.phase = Phase::BeforeData;
223                        }
224                        Phase::Data => {
225                            sink.event(Event::EndData);
226                            self.phase = Phase::BeforeEnd;
227                        }
228                        _ => {
229                            return Err(StepError::syntax(token.span, "ENDSEC outside a section"));
230                        }
231                    }
232                }
233                Token::Name(name) if name.eq_ignore_ascii_case(b"END-ISO-10303-21") => {
234                    if self.phase != Phase::BeforeEnd {
235                        return Err(StepError::syntax(
236                            token.span,
237                            "unexpected END-ISO-10303-21 marker",
238                        ));
239                    }
240                    self.expect_semicolon("after END-ISO-10303-21")?;
241                    self.phase = Phase::Done;
242                }
243                Token::Name(name) if self.phase == Phase::Header => {
244                    self.validate_header_record(&name, token.span)?;
245                    let parameters = self.parse_arguments()?;
246                    self.expect_semicolon("after header record")?;
247                    sink.event(Event::HeaderRecord(HeaderRecord {
248                        name: upper(&name),
249                        parameters,
250                    }));
251                }
252                Token::Id(id) if self.phase == Phase::Data => {
253                    let start = token.span.start;
254                    match self.parse_data_record(&id, token.span) {
255                        Ok(record) => sink.event(Event::DataRecord(record)),
256                        Err(error) => self.recover_or_fail(error, Some(start))?,
257                    }
258                }
259                _ => {
260                    let error = StepError::syntax(
261                        token.span,
262                        format!("unexpected token {:?} in {:?}", token.value, self.phase),
263                    );
264                    self.recover_or_fail(error, Some(token.span.start))?;
265                }
266            }
267        }
268        if self.phase != Phase::Done {
269            let detail = if matches!(self.phase, Phase::Header | Phase::Data) {
270                "unterminated section (expected ENDSEC)"
271            } else {
272                "physical file requires start, HEADER, DATA, and end markers"
273            };
274            return Err(StepError::syntax(self.eof_span(), detail));
275        }
276        Ok(())
277    }
278
279    /// Parses one `#id = ...;` data record, assuming the id token was consumed.
280    fn parse_data_record(
281        &mut self,
282        id: &[u8],
283        id_span: Span,
284    ) -> Result<DataRecord<String>, StepError> {
285        self.expect_equals()?;
286        let record_token = self.next()?.ok_or_else(|| {
287            StepError::syntax(Span::new(id_span.end, id_span.end), "missing record body")
288        })?;
289        let records = match record_token.value {
290            Token::Name(name) => vec![self.parse_named_record(&name)?],
291            Token::OpenParen => {
292                let mut records = Vec::new();
293                loop {
294                    if !self
295                        .peek()?
296                        .is_some_and(|next| next.value != Token::CloseParen)
297                    {
298                        break;
299                    }
300                    let component = self.next()?.ok_or_else(|| {
301                        StepError::syntax(self.eof_span(), "missing complex record")
302                    })?;
303                    let Token::Name(name) = component.value else {
304                        return Err(StepError::syntax(
305                            component.span,
306                            "expected complex record name",
307                        ));
308                    };
309                    records.push(self.parse_named_record(&name)?);
310                }
311                let close = self.next()?.ok_or_else(|| {
312                    StepError::syntax(self.eof_span(), "unterminated complex instance")
313                })?;
314                if close.value != Token::CloseParen {
315                    return Err(StepError::syntax(
316                        close.span,
317                        "expected ')' after complex instance",
318                    ));
319                }
320                if records.is_empty() {
321                    return Err(StepError::syntax(
322                        record_token.span,
323                        "complex instance must contain a record",
324                    ));
325                }
326                records
327            }
328            _ => {
329                return Err(StepError::syntax(
330                    record_token.span,
331                    "expected record name or complex instance",
332                ));
333            }
334        };
335        self.expect_semicolon("after data record")?;
336        Ok(DataRecord {
337            id: InstanceId::new(std::str::from_utf8(id).expect("instance digits are ASCII"))
338                .expect("lexer validates instance ids"),
339            records,
340        })
341    }
342
343    /// Applies the malformed-record policy.
344    ///
345    /// Recovery is deliberately narrow. It applies only inside `DATA`, only
346    /// when the caller opted in, and it always advances the cursor, so a
347    /// damaged file cannot loop. Header and section-structure defects stay
348    /// fatal under every policy: they describe the file, not one payload
349    /// record, and silently continuing past them would produce a model whose
350    /// provenance is unknown.
351    fn recover_or_fail(
352        &mut self,
353        error: StepError,
354        record_start: Option<usize>,
355    ) -> Result<(), StepError> {
356        if self.options.on_malformed_record != OnMalformed::Skip || self.phase != Phase::Data {
357            return Err(error);
358        }
359        let start = record_start.unwrap_or_else(|| error.span().start);
360        let failure = error
361            .span()
362            .end
363            .max(self.lexer.offset())
364            .max(start.saturating_add(1));
365        let resume = self.resync_from(failure);
366        self.lookahead = None;
367        self.lexer.resume_at(resume);
368        self.last_end = resume;
369        self.diagnostics.push(Diagnostic::skipped_record(
370            Span::new(start, resume),
371            format!("skipped malformed data record: {}", error.detail()),
372        ));
373        Ok(())
374    }
375
376    /// Finds the next byte offset at which parsing can safely restart.
377    ///
378    /// Scans raw bytes rather than tokens because the tokenizer is what
379    /// failed. Quoted strings and comments are tracked so a `;` or `ENDSEC`
380    /// inside a literal is not mistaken for a record boundary. A section
381    /// terminator stops the scan *before* it is consumed, so recovery can
382    /// never swallow the end of `DATA`.
383    fn resync_from(&self, from: usize) -> usize {
384        let mut position = from.min(self.input.len());
385        let mut in_string = false;
386        while position < self.input.len() {
387            let byte = self.input[position];
388            if in_string {
389                if byte == b'\'' {
390                    if self.input.get(position + 1) == Some(&b'\'') {
391                        position += 2;
392                        continue;
393                    }
394                    in_string = false;
395                }
396                position += 1;
397                continue;
398            }
399            match byte {
400                b'\'' => {
401                    in_string = true;
402                    position += 1;
403                }
404                b'/' if self.input.get(position + 1) == Some(&b'*') => {
405                    position = self.input[position + 2..]
406                        .windows(2)
407                        .position(|window| window == b"*/")
408                        .map_or(self.input.len(), |offset| position + 2 + offset + 2);
409                }
410                b';' => return position + 1,
411                _ if self.section_end_at(position) => return position,
412                _ => position += 1,
413            }
414        }
415        self.input.len()
416    }
417
418    fn section_end_at(&self, position: usize) -> bool {
419        let preceded_by_word = position
420            .checked_sub(1)
421            .and_then(|previous| self.input.get(previous))
422            .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_');
423        !preceded_by_word
424            && self
425                .input
426                .get(position..position + 6)
427                .is_some_and(|bytes| bytes.eq_ignore_ascii_case(b"ENDSEC"))
428    }
429
430    fn validate_header_record(&mut self, name: &[u8], span: Span) -> Result<(), StepError> {
431        const REQUIRED: [&[u8]; 3] = [b"FILE_DESCRIPTION", b"FILE_NAME", b"FILE_SCHEMA"];
432        if let Some(expected) = REQUIRED.get(self.header_records_seen) {
433            if !name.eq_ignore_ascii_case(expected) {
434                return Err(StepError::syntax(
435                    span,
436                    format!(
437                        "expected mandatory {} header record",
438                        String::from_utf8_lossy(expected)
439                    ),
440                ));
441            }
442        } else if REQUIRED
443            .iter()
444            .any(|required| name.eq_ignore_ascii_case(required))
445        {
446            return Err(StepError::syntax(span, "duplicate mandatory header record"));
447        }
448        self.header_records_seen += 1;
449        Ok(())
450    }
451
452    fn parse_named_record(&mut self, name: &[u8]) -> Result<Record, StepError> {
453        Ok(Record {
454            name: upper(name),
455            parameters: self.parse_arguments()?,
456        })
457    }
458
459    fn parse_arguments(&mut self) -> Result<Vec<Parameter>, StepError> {
460        let token = self
461            .next()?
462            .ok_or_else(|| StepError::syntax(self.eof_span(), "expected '(' after record name"))?;
463        if token.value != Token::OpenParen {
464            return Err(StepError::syntax(
465                token.span,
466                "expected '(' after record name",
467            ));
468        }
469        self.parse_parameter_list(0)
470    }
471
472    fn parse_parameter_list(&mut self, depth: usize) -> Result<Vec<Parameter>, StepError> {
473        if depth > crate::MAX_PARAMETER_NESTING {
474            let span = match self.peek()? {
475                Some(token) => token.span,
476                None => self.eof_span(),
477            };
478            return Err(StepError::syntax(span, "parameter nesting limit exceeded"));
479        }
480        let mut parameters = Vec::new();
481        if self
482            .peek()?
483            .is_some_and(|token| token.value == Token::CloseParen)
484        {
485            let _ = self.next()?;
486            return Ok(parameters);
487        }
488        loop {
489            parameters.push(self.parse_parameter(depth)?);
490            let separator = self
491                .next()?
492                .ok_or_else(|| StepError::syntax(self.eof_span(), "unterminated parameter list"))?;
493            match separator.value {
494                Token::Comma => {}
495                Token::CloseParen => return Ok(parameters),
496                _ => {
497                    return Err(StepError::syntax(
498                        separator.span,
499                        "expected ',' or ')' after parameter",
500                    ));
501                }
502            }
503        }
504    }
505
506    fn parse_parameter(&mut self, depth: usize) -> Result<Parameter, StepError> {
507        let token = self
508            .next()?
509            .ok_or_else(|| StepError::syntax(self.eof_span(), "expected parameter"))?;
510        match token.value {
511            Token::Dollar => Ok(Parameter::Null),
512            Token::Star => Ok(Parameter::Derived),
513            Token::Id(id) => Ok(Parameter::Ref(
514                InstanceId::new(std::str::from_utf8(&id).expect("instance digits are ASCII"))
515                    .expect("lexer validates instance ids"),
516            )),
517            Token::Integer(value) => Ok(Parameter::Integer(
518                String::from_utf8_lossy(&value).into_owned(),
519            )),
520            Token::Real(value) => Ok(Parameter::Real(
521                String::from_utf8_lossy(&value).into_owned(),
522            )),
523            Token::Text(raw) => Ok(Parameter::Text(escape::decode(&raw))),
524            Token::Binary(raw) => Ok(Parameter::Binary(
525                String::from_utf8_lossy(&raw).into_owned(),
526            )),
527            Token::Keyword(keyword) if keyword.eq_ignore_ascii_case(b"T") => {
528                Ok(Parameter::Bool(true))
529            }
530            Token::Keyword(keyword) if keyword.eq_ignore_ascii_case(b"F") => {
531                Ok(Parameter::Bool(false))
532            }
533            Token::Keyword(keyword) if keyword.eq_ignore_ascii_case(b"U") => {
534                Ok(Parameter::LogicalUnknown)
535            }
536            Token::Keyword(keyword) => Ok(Parameter::Enum(upper(&keyword))),
537            Token::OpenParen => Ok(Parameter::List(self.parse_parameter_list(depth + 1)?)),
538            Token::Name(name) => {
539                let Some(next) = self.peek()? else {
540                    return Err(StepError::syntax(
541                        self.eof_span(),
542                        "expected '(' after typed parameter name",
543                    ));
544                };
545                if next.value != Token::OpenParen {
546                    return Err(StepError::syntax(
547                        next.span,
548                        "expected '(' after typed parameter name",
549                    ));
550                }
551                let _ = self.next()?;
552                let mut parameters = self.parse_parameter_list(depth + 1)?;
553                let value = if parameters.len() == 1 {
554                    Box::new(parameters.remove(0))
555                } else {
556                    Box::new(Parameter::List(parameters))
557                };
558                Ok(Parameter::Typed {
559                    type_name: upper(&name),
560                    value,
561                })
562            }
563            value => Err(StepError::syntax(
564                token.span,
565                format!("unexpected parameter token {value:?}"),
566            )),
567        }
568    }
569
570    fn expect_equals(&mut self) -> Result<(), StepError> {
571        let token = self
572            .next()?
573            .ok_or_else(|| StepError::syntax(self.eof_span(), "expected '=' after instance id"))?;
574        if token.value == Token::Equals {
575            Ok(())
576        } else {
577            Err(StepError::syntax(
578                token.span,
579                "expected '=' after instance id",
580            ))
581        }
582    }
583
584    fn expect_semicolon(&mut self, context: &str) -> Result<(), StepError> {
585        let token = self
586            .next()?
587            .ok_or_else(|| StepError::syntax(self.eof_span(), format!("expected ';' {context}")))?;
588        if token.value == Token::Semicolon {
589            Ok(())
590        } else {
591            Err(StepError::syntax(
592                token.span,
593                format!("expected ';' {context}"),
594            ))
595        }
596    }
597
598    fn next(&mut self) -> Result<Option<Spanned<Token<'a>>>, StepError> {
599        let token = match self.lookahead.take() {
600            Some(token) => Some(token),
601            None => self.lexer.next_spanned()?,
602        };
603        if let Some(token) = &token {
604            self.last_end = token.span.end;
605        }
606        Ok(token)
607    }
608
609    fn peek(&mut self) -> Result<Option<&Spanned<Token<'a>>>, StepError> {
610        if self.lookahead.is_none() {
611            self.lookahead = self.lexer.next_spanned()?;
612        }
613        Ok(self.lookahead.as_ref())
614    }
615
616    fn eof_span(&self) -> Span {
617        let offset = self.last_end.max(self.lexer.offset());
618        Span::new(offset, offset)
619    }
620}
621
622fn upper(bytes: &[u8]) -> String {
623    String::from_utf8_lossy(bytes).to_ascii_uppercase()
624}
openbim_step/parser.rs

openbim_step/
parser.rs
