openbim_step/
parser.rs

1//! Semantic parser and event/sink interface.
2//!
3//! Parsing is structural: record names and parameters are retained without a
4//! domain schema. Unknown header and data records therefore survive a
5//! parse/write/reparse cycle.
6
7use crate::escape;
8use crate::lexer::{Lexer, Token};
9use crate::recovery::{Diagnostic, OnMalformed, ParseOptions, ParseOutcome};
10use crate::{
11    DataRecord, DataSection, Exchange, HeaderRecord, HeaderSection, InstanceId, Parameter, Record,
12    Span, Spanned, StepError,
13};
14
15/// A semantic parser event.
16#[derive(Debug, Clone, PartialEq)]
17pub enum Event<S = String> {
18    /// Entered `HEADER;`.
19    StartHeader,
20    /// Parsed one header record.
21    HeaderRecord(HeaderRecord<S>),
22    /// Reached the header `ENDSEC;`.
23    EndHeader,
24    /// Entered `DATA;`.
25    StartData,
26    /// Parsed one data record.
27    DataRecord(DataRecord<S>),
28    /// Reached the data `ENDSEC;`.
29    EndData,
30}
31
32/// Consumer for parse events.
33pub trait EventSink<S = String> {
34    /// Receives one event. Events are delivered in source order.
35    fn event(&mut self, event: Event<S>);
36}
37
38impl<S, F> EventSink<S> for F
39where
40    F: FnMut(Event<S>),
41{
42    fn event(&mut self, event: Event<S>) {
43        self(event);
44    }
45}
46
47/// Parses a physical file into an owned generic exchange structure.
48/// # Errors
49///
50/// Returns [`StepError`] for the wrong
51/// physical-file marker, or a spanned lexical/syntax diagnostic otherwise.
52pub fn parse(input: &[u8]) -> Result<Exchange, StepError> {
53    parse_with(input, ParseOptions::strict()).map(|outcome| outcome.exchange)
54}
55
56/// Parses a physical file under explicit [`ParseOptions`].
57///
58/// With [`OnMalformed::Skip`] an unparsable data record is reported as a
59/// [`Diagnostic`] and the parser resynchronizes on the next record or section
60/// end, so a consumer can load a damaged file and still show exactly what was
61/// dropped. Header structure stays strict under every policy: mandatory header
62/// records are file-level invariants, not recoverable payload.
63/// # Errors
64///
65/// Returns [`StepError`] for the wrong physical-file marker, for any header or
66/// section-structure defect, and for data defects when the policy is
67/// [`OnMalformed::Abort`].
68pub fn parse_with(input: &[u8], options: ParseOptions) -> Result<ParseOutcome, StepError> {
69    #[derive(Default)]
70    struct Builder {
71        header: HeaderSection,
72        data: DataSection,
73    }
74
75    impl EventSink for Builder {
76        fn event(&mut self, event: Event) {
77            match event {
78                Event::HeaderRecord(record) => self.header.records.push(record),
79                Event::DataRecord(record) => self.data.records.push(record),
80                Event::StartHeader | Event::EndHeader | Event::StartData | Event::EndData => {}
81            }
82        }
83    }
84
85    let mut builder = Builder::default();
86    let diagnostics = parse_events_with(input, &mut builder, options)?;
87    Ok(ParseOutcome {
88        exchange: Exchange {
89            header: builder.header,
90            data: builder.data,
91        },
92        diagnostics,
93    })
94}
95
96/// Parses a physical file and sends semantic records to `sink`.
97///
98/// Unlike [`parse`], this API does not accumulate an [`Exchange`]. It is useful
99/// for import pipelines that index, validate, or transform records as they are
100/// read.
101/// # Errors
102///
103/// Returns the same physical-file and syntax diagnostics as [`parse`].
104pub fn parse_events(input: &[u8], sink: &mut impl EventSink) -> Result<(), StepError> {
105    parse_events_with(input, sink, ParseOptions::strict()).map(|_| ())
106}
107
108/// Streams semantic records under explicit [`ParseOptions`], returning the
109/// non-fatal diagnostics collected on the way.
110/// # Errors
111///
112/// Returns the same physical-file, header, and structure diagnostics as
113/// [`parse_with`].
114pub fn parse_events_with(
115    input: &[u8],
116    sink: &mut impl EventSink,
117    options: ParseOptions,
118) -> Result<Vec<Diagnostic>, StepError> {
119    if !crate::is_step_file(input) {
120        return Err(StepError::not_step("missing ISO-10303-21 marker"));
121    }
122    let mut parser = Parser::new(input);
123    parser.options = options;
124    parser.parse(sink)?;
125    Ok(parser.diagnostics)
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq)]
129enum Phase {
130    BeforeStart,
131    BeforeHeader,
132    Header,
133    BeforeData,
134    Data,
135    BeforeEnd,
136    Done,
137}
138
139struct Parser<'a> {
140    input: &'a [u8],
141    lexer: Lexer<'a>,
142    lookahead: Option<Spanned<Token<'a>>>,
143    last_end: usize,
144    phase: Phase,
145    header_records_seen: usize,
146    options: ParseOptions,
147    diagnostics: Vec<Diagnostic>,
148}
149
150impl<'a> Parser<'a> {
151    fn new(input: &'a [u8]) -> Self {
152        Self {
153            input,
154            lexer: Lexer::new(input),
155            lookahead: None,
156            last_end: 0,
157            phase: Phase::BeforeStart,
158            header_records_seen: 0,
159            options: ParseOptions::strict(),
160            diagnostics: Vec::new(),
161        }
162    }
163
164    // Keeping section dispatch together makes the state-machine transitions auditable.
165    #[allow(clippy::too_many_lines)]
166    fn parse(&mut self, sink: &mut impl EventSink) -> Result<(), StepError> {
167        loop {
168            let token = match self.next() {
169                Ok(Some(token)) => token,
170                Ok(None) => break,
171                Err(error) => {
172                    // A lexical defect inside DATA is recoverable: the damaged
173                    // bytes belong to one record, not to the file structure.
174                    self.recover_or_fail(error, None)?;
175                    continue;
176                }
177            };
178            if self.phase == Phase::Done {
179                return Err(StepError::syntax(
180                    token.span,
181                    "content after END-ISO-10303-21",
182                ));
183            }
184            match token.value {
185                Token::Name(name) if name.eq_ignore_ascii_case(b"ISO-10303-21") => {
186                    if self.phase != Phase::BeforeStart {
187                        return Err(StepError::syntax(
188                            token.span,
189                            "unexpected ISO-10303-21 marker",
190                        ));
191                    }
192                    self.expect_semicolon("after ISO-10303-21")?;
193                    self.phase = Phase::BeforeHeader;
194                }
195                Token::Name(name) if name.eq_ignore_ascii_case(b"HEADER") => {
196                    if self.phase != Phase::BeforeHeader {
197                        return Err(StepError::syntax(token.span, "unexpected HEADER section"));
198                    }
199                    self.expect_semicolon("after HEADER")?;
200                    self.phase = Phase::Header;
201                    sink.event(Event::StartHeader);
202                }
203                Token::Name(name) if name.eq_ignore_ascii_case(b"DATA") => {
204                    if self.phase != Phase::BeforeData {
205                        return Err(StepError::syntax(token.span, "unexpected DATA section"));
206                    }
207                    self.expect_semicolon("after DATA")?;
208                    self.phase = Phase::Data;
209                    sink.event(Event::StartData);
210                }
211                Token::Name(name) if name.eq_ignore_ascii_case(b"ENDSEC") => {
212                    self.expect_semicolon("after ENDSEC")?;
213                    match self.phase {
214                        Phase::Header => {
215                            if self.header_records_seen < 3 {
216                                return Err(StepError::syntax(
217                                    token.span,
218                                    "missing mandatory STEP header record",
219                                ));
220                            }
221                            sink.event(Event::EndHeader);
222                            self.phase = Phase::BeforeData;
223                        }
224                        Phase::Data => {
225                            sink.event(Event::EndData);
226                            self.phase = Phase::BeforeEnd;
227                        }
228                        _ => {
229                            return Err(StepError::syntax(token.span, "ENDSEC outside a section"));
230                        }
231                    }
232                }
233                Token::Name(name) if name.eq_ignore_ascii_case(b"END-ISO-10303-21") => {
234                    if self.phase != Phase::BeforeEnd {
235                        return Err(StepError::syntax(
236                            token.span,
237                            "unexpected END-ISO-10303-21 marker",
238                        ));
239                    }
240                    self.expect_semicolon("after END-ISO-10303-21")?;
241                    self.phase = Phase::Done;
242                }
243                Token::Name(name) if self.phase == Phase::Header => {
244                    self.validate_header_record(&name, token.span)?;
245                    let parameters = self.parse_arguments()?;
246                    self.expect_semicolon("after header record")?;
247                    sink.event(Event::HeaderRecord(HeaderRecord {
248                        name: upper(&name),
249                        parameters,
250                    }));
251                }
252                Token::Id(id) if self.phase == Phase::Data => {
253                    let start = token.span.start;
254                    match self.parse_data_record(&id, token.span) {
255                        Ok(record) => sink.event(Event::DataRecord(record)),
256                        Err(error) => self.recover_or_fail(error, Some(start))?,
257                    }
258                }
259                _ => {
260                    let error = StepError::syntax(
261                        token.span,
262                        format!("unexpected token {:?} in {:?}", token.value, self.phase),
263                    );
264                    self.recover_or_fail(error, Some(token.span.start))?;
265                }
266            }
267        }
268        if self.phase != Phase::Done {
269            let detail = if matches!(self.phase, Phase::Header | Phase::Data) {
270                "unterminated section (expected ENDSEC)"
271            } else {
272                "physical file requires start, HEADER, DATA, and end markers"
273            };
274            return Err(StepError::syntax(self.eof_span(), detail));
275        }
276        Ok(())
277    }
278
279    /// Parses one `#id = ...;` data record, assuming the id token was consumed.
280    fn parse_data_record(
281        &mut self,
282        id: &[u8],
283        id_span: Span,
284    ) -> Result<DataRecord<String>, StepError> {
285        self.expect_equals()?;
286        let record_token = self.next()?.ok_or_else(|| {
287            StepError::syntax(Span::new(id_span.end, id_span.end), "missing record body")
288        })?;
289        let records = match record_token.value {
290            Token::Name(name) => vec![self.parse_named_record(&name)?],
291            Token::OpenParen => {
292                let mut records = Vec::new();
293                loop {
294                    if !self
295                        .peek()?
296                        .is_some_and(|next| next.value != Token::CloseParen)
297                    {
298                        break;
299                    }
300                    let component = self.next()?.ok_or_else(|| {
301                        StepError::syntax(self.eof_span(), "missing complex record")
302                    })?;
303                    let Token::Name(name) = component.value else {
304                        return Err(StepError::syntax(
305                            component.span,
306                            "expected complex record name",
307                        ));
308                    };
309                    records.push(self.parse_named_record(&name)?);
310                }
311                let close = self.next()?.ok_or_else(|| {
312                    StepError::syntax(self.eof_span(), "unterminated complex instance")
313                })?;
314                if close.value != Token::CloseParen {
315                    return Err(StepError::syntax(
316                        close.span,
317                        "expected ')' after complex instance",
318                    ));
319                }
320                if records.is_empty() {
321                    return Err(StepError::syntax(
322                        record_token.span,
323                        "complex instance must contain a record",
324                    ));
325                }
326                records
327            }
328            _ => {
329                return Err(StepError::syntax(
330                    record_token.span,
331                    "expected record name or complex instance",
332                ));
333            }
334        };
335        self.expect_semicolon("after data record")?;
336        Ok(DataRecord {
337            id: InstanceId::new(std::str::from_utf8(id).expect("instance digits are ASCII"))
338                .expect("lexer validates instance ids"),
339            records,
340        })
341    }
342
343    /// Applies the malformed-record policy.
344    ///
345    /// Recovery is deliberately narrow. It applies only inside `DATA`, only
346    /// when the caller opted in, and it always advances the cursor, so a
347    /// damaged file cannot loop. Header and section-structure defects stay
348    /// fatal under every policy: they describe the file, not one payload
349    /// record, and silently continuing past them would produce a model whose
350    /// provenance is unknown.
351    fn recover_or_fail(
352        &mut self,
353        error: StepError,
354        record_start: Option<usize>,
355    ) -> Result<(), StepError> {
356        if self.options.on_malformed_record != OnMalformed::Skip || self.phase != Phase::Data {
357            return Err(error);
358        }
359        let start = record_start.unwrap_or_else(|| error.span().start);
360        // Resynchronize from just past the record's first byte, NOT from the
361        // end of the error span. A diagnostic can legitimately span the token
362        // that follows the damage -- including `ENDSEC` -- and resuming past
363        // it would swallow the section terminator.
364        let resume = self.resync_from(start.saturating_add(1));
365        self.lookahead = None;
366        self.lexer.resume_at(resume);
367        self.last_end = resume;
368        self.diagnostics.push(Diagnostic::skipped_record(
369            Span::new(start, resume),
370            format!("skipped malformed data record: {}", error.detail()),
371        ));
372        Ok(())
373    }
374
375    /// Finds the next byte offset at which parsing can safely restart.
376    ///
377    /// Scans raw bytes rather than tokens because the tokenizer is what
378    /// failed. All three literal kinds -- quoted strings, binary literals, and
379    /// comments -- are tracked, so a `;` or `ENDSEC` inside a literal is not
380    /// mistaken for a record boundary. Scanning a literal's payload as code
381    /// would let recovery fabricate records that were never in the source.
382    /// A section terminator stops the scan *before* it is consumed, so
383    /// recovery can never swallow the end of `DATA`.
384    fn resync_from(&self, from: usize) -> usize {
385        // Which literal the scanner is currently inside. STEP has three, and
386        // all of them can contain bytes that look like record syntax.
387        enum Literal {
388            None,
389            // `'...'`, where `''` is an escaped apostrophe rather than a close.
390            Text,
391            // `"...."`, with no doubling rule: the first `"` closes it.
392            Binary,
393        }
394
395        let mut position = from.min(self.input.len());
396        let mut literal = Literal::None;
397        while position < self.input.len() {
398            let byte = self.input[position];
399            match literal {
400                Literal::Text => {
401                    if byte == b'\'' {
402                        if self.input.get(position + 1) == Some(&b'\'') {
403                            position += 2;
404                            continue;
405                        }
406                        literal = Literal::None;
407                    }
408                    position += 1;
409                }
410                Literal::Binary => {
411                    if byte == b'"' {
412                        literal = Literal::None;
413                    }
414                    position += 1;
415                }
416                Literal::None => match byte {
417                    b'\'' => {
418                        literal = Literal::Text;
419                        position += 1;
420                    }
421                    b'"' => {
422                        literal = Literal::Binary;
423                        position += 1;
424                    }
425                    b'/' if self.input.get(position + 1) == Some(&b'*') => {
426                        position = self.input[position + 2..]
427                            .windows(2)
428                            .position(|window| window == b"*/")
429                            .map_or(self.input.len(), |offset| position + 2 + offset + 2);
430                    }
431                    b';' => return position + 1,
432                    _ if self.section_end_at(position) => return position,
433                    _ => position += 1,
434                },
435            }
436        }
437        self.input.len()
438    }
439
440    fn section_end_at(&self, position: usize) -> bool {
441        let preceded_by_word = position
442            .checked_sub(1)
443            .and_then(|previous| self.input.get(previous))
444            .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_');
445        !preceded_by_word
446            && self
447                .input
448                .get(position..position + 6)
449                .is_some_and(|bytes| bytes.eq_ignore_ascii_case(b"ENDSEC"))
450    }
451
452    fn validate_header_record(&mut self, name: &[u8], span: Span) -> Result<(), StepError> {
453        const REQUIRED: [&[u8]; 3] = [b"FILE_DESCRIPTION", b"FILE_NAME", b"FILE_SCHEMA"];
454        if let Some(expected) = REQUIRED.get(self.header_records_seen) {
455            if !name.eq_ignore_ascii_case(expected) {
456                return Err(StepError::syntax(
457                    span,
458                    format!(
459                        "expected mandatory {} header record",
460                        String::from_utf8_lossy(expected)
461                    ),
462                ));
463            }
464        } else if REQUIRED
465            .iter()
466            .any(|required| name.eq_ignore_ascii_case(required))
467        {
468            return Err(StepError::syntax(span, "duplicate mandatory header record"));
469        }
470        self.header_records_seen += 1;
471        Ok(())
472    }
473
474    fn parse_named_record(&mut self, name: &[u8]) -> Result<Record, StepError> {
475        Ok(Record {
476            name: upper(name),
477            parameters: self.parse_arguments()?,
478        })
479    }
480
481    fn parse_arguments(&mut self) -> Result<Vec<Parameter>, StepError> {
482        let token = self
483            .next()?
484            .ok_or_else(|| StepError::syntax(self.eof_span(), "expected '(' after record name"))?;
485        if token.value != Token::OpenParen {
486            return Err(StepError::syntax(
487                token.span,
488                "expected '(' after record name",
489            ));
490        }
491        self.parse_parameter_list(0)
492    }
493
494    fn parse_parameter_list(&mut self, depth: usize) -> Result<Vec<Parameter>, StepError> {
495        if depth > crate::MAX_PARAMETER_NESTING {
496            let span = match self.peek()? {
497                Some(token) => token.span,
498                None => self.eof_span(),
499            };
500            return Err(StepError::syntax(span, "parameter nesting limit exceeded"));
501        }
502        let mut parameters = Vec::new();
503        if self
504            .peek()?
505            .is_some_and(|token| token.value == Token::CloseParen)
506        {
507            let _ = self.next()?;
508            return Ok(parameters);
509        }
510        loop {
511            parameters.push(self.parse_parameter(depth)?);
512            let separator = self
513                .next()?
514                .ok_or_else(|| StepError::syntax(self.eof_span(), "unterminated parameter list"))?;
515            match separator.value {
516                Token::Comma => {}
517                Token::CloseParen => return Ok(parameters),
518                _ => {
519                    return Err(StepError::syntax(
520                        separator.span,
521                        "expected ',' or ')' after parameter",
522                    ));
523                }
524            }
525        }
526    }
527
528    fn parse_parameter(&mut self, depth: usize) -> Result<Parameter, StepError> {
529        let token = self
530            .next()?
531            .ok_or_else(|| StepError::syntax(self.eof_span(), "expected parameter"))?;
532        match token.value {
533            Token::Dollar => Ok(Parameter::Null),
534            Token::Star => Ok(Parameter::Derived),
535            Token::Id(id) => Ok(Parameter::Ref(
536                InstanceId::new(std::str::from_utf8(&id).expect("instance digits are ASCII"))
537                    .expect("lexer validates instance ids"),
538            )),
539            Token::Integer(value) => Ok(Parameter::Integer(
540                String::from_utf8_lossy(&value).into_owned(),
541            )),
542            Token::Real(value) => Ok(Parameter::Real(
543                String::from_utf8_lossy(&value).into_owned(),
544            )),
545            Token::Text(raw) => Ok(Parameter::Text(escape::decode(&raw))),
546            Token::Binary(raw) => Ok(Parameter::Binary(
547                String::from_utf8_lossy(&raw).into_owned(),
548            )),
549            Token::Keyword(keyword) if keyword.eq_ignore_ascii_case(b"T") => {
550                Ok(Parameter::Bool(true))
551            }
552            Token::Keyword(keyword) if keyword.eq_ignore_ascii_case(b"F") => {
553                Ok(Parameter::Bool(false))
554            }
555            Token::Keyword(keyword) if keyword.eq_ignore_ascii_case(b"U") => {
556                Ok(Parameter::LogicalUnknown)
557            }
558            Token::Keyword(keyword) => Ok(Parameter::Enum(upper(&keyword))),
559            Token::OpenParen => Ok(Parameter::List(self.parse_parameter_list(depth + 1)?)),
560            Token::Name(name) => {
561                let Some(next) = self.peek()? else {
562                    return Err(StepError::syntax(
563                        self.eof_span(),
564                        "expected '(' after typed parameter name",
565                    ));
566                };
567                if next.value != Token::OpenParen {
568                    return Err(StepError::syntax(
569                        next.span,
570                        "expected '(' after typed parameter name",
571                    ));
572                }
573                let _ = self.next()?;
574                let mut parameters = self.parse_parameter_list(depth + 1)?;
575                let value = if parameters.len() == 1 {
576                    Box::new(parameters.remove(0))
577                } else {
578                    Box::new(Parameter::List(parameters))
579                };
580                Ok(Parameter::Typed {
581                    type_name: upper(&name),
582                    value,
583                })
584            }
585            value => Err(StepError::syntax(
586                token.span,
587                format!("unexpected parameter token {value:?}"),
588            )),
589        }
590    }
591
592    fn expect_equals(&mut self) -> Result<(), StepError> {
593        let token = self
594            .next()?
595            .ok_or_else(|| StepError::syntax(self.eof_span(), "expected '=' after instance id"))?;
596        if token.value == Token::Equals {
597            Ok(())
598        } else {
599            Err(StepError::syntax(
600                token.span,
601                "expected '=' after instance id",
602            ))
603        }
604    }
605
606    fn expect_semicolon(&mut self, context: &str) -> Result<(), StepError> {
607        let token = self
608            .next()?
609            .ok_or_else(|| StepError::syntax(self.eof_span(), format!("expected ';' {context}")))?;
610        if token.value == Token::Semicolon {
611            Ok(())
612        } else {
613            Err(StepError::syntax(
614                token.span,
615                format!("expected ';' {context}"),
616            ))
617        }
618    }
619
620    fn next(&mut self) -> Result<Option<Spanned<Token<'a>>>, StepError> {
621        let token = match self.lookahead.take() {
622            Some(token) => Some(token),
623            None => self.lexer.next_spanned()?,
624        };
625        if let Some(token) = &token {
626            self.last_end = token.span.end;
627        }
628        Ok(token)
629    }
630
631    fn peek(&mut self) -> Result<Option<&Spanned<Token<'a>>>, StepError> {
632        if self.lookahead.is_none() {
633            self.lookahead = self.lexer.next_spanned()?;
634        }
635        Ok(self.lookahead.as_ref())
636    }
637
638    fn eof_span(&self) -> Span {
639        let offset = self.last_end.max(self.lexer.offset());
640        Span::new(offset, offset)
641    }
642}
643
644fn upper(bytes: &[u8]) -> String {
645    String::from_utf8_lossy(bytes).to_ascii_uppercase()
646}
openbim_step/parser.rs

openbim_step/
parser.rs
