edifact_parser/
parser.rs

1#[cfg(test)]
2use edifact_primitives::Control;
3use edifact_primitives::{EdifactDelimiters, RawSegment, SegmentPosition};
4
5use crate::error::ParseError;
6use crate::handler::EdifactHandler;
7use crate::segment_builder::SegmentBuilder as SegBuilder;
8use crate::tokenizer::EdifactTokenizer;
9
10/// Streaming EDIFACT parser.
11///
12/// Parses a byte slice by tokenizing it into segments and routing them
13/// to an `EdifactHandler`. Service segments (UNB, UNH, UNT, UNZ) are
14/// dispatched to specific handler methods in addition to `on_segment()`.
15pub struct EdifactStreamParser;
16
17impl EdifactStreamParser {
18    /// Parse an EDIFACT interchange from a byte slice.
19    ///
20    /// This is the main synchronous entry point. It:
21    /// 1. Detects UNA and determines delimiters
22    /// 2. Tokenizes input into segments
23    /// 3. Routes each segment to the handler
24    /// 4. Stops if the handler returns `Control::Stop`
25    ///
26    /// Supports both UTF-8 and ISO-8859-1 encoded input. If the input
27    /// contains non-UTF-8 bytes (raw ISO-8859-1), it is transparently
28    /// transcoded to UTF-8 before parsing. EDIFACT delimiters are always
29    /// ASCII, so transcoding does not affect delimiter detection.
30    pub fn parse(input: &[u8], handler: &mut dyn EdifactHandler) -> Result<(), ParseError> {
31        if std::str::from_utf8(input).is_ok() {
32            Self::parse_inner(input, handler)
33        } else {
34            let transcoded = transcode_iso_8859_1_to_utf8(input);
35            Self::parse_inner(&transcoded, handler)
36        }
37    }
38
39    fn parse_inner(input: &[u8], handler: &mut dyn EdifactHandler) -> Result<(), ParseError> {
40        // Step 1: Detect delimiters
41        let (has_una, delimiters) = EdifactDelimiters::detect(input);
42        handler.on_delimiters(&delimiters, has_una);
43
44        // Step 2: Determine where actual content starts (after UNA if present)
45        let content_start = if has_una { 9 } else { 0 };
46        let content = &input[content_start..];
47
48        // Step 3: Tokenize and process segments
49        let tokenizer = EdifactTokenizer::new(delimiters);
50        let seg_builder = SegBuilder::new(delimiters);
51
52        let mut segment_number: u32 = 0;
53        let mut message_number: u32 = 0;
54        let mut byte_offset = content_start;
55
56        for segment_str in tokenizer.tokenize_segments(content) {
57            segment_number += 1;
58
59            let position = SegmentPosition::new(segment_number, byte_offset, message_number);
60
61            let Some(raw_segment) = seg_builder.build(segment_str, position) else {
62                byte_offset += segment_str.len() + 1; // +1 for terminator
63                continue;
64            };
65
66            // Skip UNA segments in content
67            if raw_segment.is("UNA") {
68                byte_offset += segment_str.len() + 1;
69                segment_number -= 1; // don't count UNA
70                continue;
71            }
72
73            let id_upper = raw_segment.id.to_ascii_uppercase();
74
75            // Track message numbering
76            if id_upper == "UNH" {
77                message_number += 1;
78            }
79
80            // Rebuild position with correct message number
81            let effective_message_number = if id_upper == "UNB" || id_upper == "UNZ" {
82                0
83            } else {
84                message_number
85            };
86            let position =
87                SegmentPosition::new(segment_number, byte_offset, effective_message_number);
88            let raw_segment = RawSegment::new(raw_segment.id, raw_segment.elements, position);
89
90            // Route service segments
91            match id_upper.as_str() {
92                // The handler runs for its side effects either way — bind the
93                // control flag first so the call does not end up in a match
94                // guard, where it would be easy to misread as pure.
95                "UNB" => {
96                    let control = handler.on_interchange_start(&raw_segment);
97                    if control.should_stop() {
98                        return Ok(());
99                    }
100                }
101                "UNH" => {
102                    let control = handler.on_message_start(&raw_segment);
103                    if control.should_stop() {
104                        return Ok(());
105                    }
106                }
107                "UNT" => {
108                    handler.on_message_end(&raw_segment);
109                }
110                "UNZ" => {
111                    handler.on_interchange_end(&raw_segment);
112                }
113                _ => {}
114            }
115
116            // Always call on_segment
117            if handler.on_segment(&raw_segment).should_stop() {
118                return Ok(());
119            }
120
121            byte_offset += segment_str.len() + 1; // +1 for terminator
122        }
123
124        Ok(())
125    }
126}
127
128/// Transcode ISO-8859-1 bytes to UTF-8.
129///
130/// ISO-8859-1 code points 0x00–0xFF map directly to Unicode U+0000–U+00FF.
131/// - 0x00–0x7F: single UTF-8 byte (identical)
132/// - 0x80–0xBF: two UTF-8 bytes: 0xC2 + original byte
133/// - 0xC0–0xFF: two UTF-8 bytes: 0xC3 + (original byte - 0x40)
134fn transcode_iso_8859_1_to_utf8(input: &[u8]) -> Vec<u8> {
135    let mut output = Vec::with_capacity(input.len() + input.len() / 4);
136    for &b in input {
137        if b < 0x80 {
138            output.push(b);
139        } else {
140            // ISO-8859-1 byte to UTF-8 two-byte sequence
141            output.push(0xC0 | (b >> 6));
142            output.push(0x80 | (b & 0x3F));
143        }
144    }
145    output
146}
147
148#[cfg(test)]
149mod tests {
150    use super::*;
151    use std::cell::RefCell;
152
153    /// Handler that collects all event names in order.
154    struct EventCollector {
155        events: RefCell<Vec<String>>,
156    }
157
158    impl EventCollector {
159        fn new() -> Self {
160            Self {
161                events: RefCell::new(Vec::new()),
162            }
163        }
164
165        fn events(&self) -> Vec<String> {
166            self.events.borrow().clone()
167        }
168    }
169
170    impl EdifactHandler for EventCollector {
171        fn on_delimiters(&mut self, _d: &EdifactDelimiters, explicit_una: bool) {
172            self.events
173                .borrow_mut()
174                .push(format!("DELIMITERS(una={})", explicit_una));
175        }
176
177        fn on_interchange_start(&mut self, unb: &RawSegment) -> Control {
178            self.events
179                .borrow_mut()
180                .push(format!("INTERCHANGE_START({})", unb.id));
181            Control::Continue
182        }
183
184        fn on_message_start(&mut self, unh: &RawSegment) -> Control {
185            self.events
186                .borrow_mut()
187                .push(format!("MESSAGE_START(ref={})", unh.get_element(0)));
188            Control::Continue
189        }
190
191        fn on_segment(&mut self, seg: &RawSegment) -> Control {
192            self.events
193                .borrow_mut()
194                .push(format!("SEGMENT({})", seg.id));
195            Control::Continue
196        }
197
198        fn on_message_end(&mut self, _unt: &RawSegment) {
199            self.events.borrow_mut().push("MESSAGE_END".to_string());
200        }
201
202        fn on_interchange_end(&mut self, _unz: &RawSegment) {
203            self.events.borrow_mut().push("INTERCHANGE_END".to_string());
204        }
205    }
206
207    #[test]
208    fn test_parse_minimal_interchange() {
209        let input = b"UNA:+.? 'UNB+UNOC:3+SENDER+RECEIVER+210101:1200+REF001'UNH+MSG001+UTILMD:D:11A:UN:S2.1'BGM+E03+DOC001'UNT+3+MSG001'UNZ+1+REF001'";
210
211        let mut handler = EventCollector::new();
212        EdifactStreamParser::parse(input, &mut handler).unwrap();
213
214        let events = handler.events();
215        assert_eq!(events[0], "DELIMITERS(una=true)");
216        assert_eq!(events[1], "INTERCHANGE_START(UNB)");
217        assert_eq!(events[2], "SEGMENT(UNB)");
218        assert_eq!(events[3], "MESSAGE_START(ref=MSG001)");
219        assert_eq!(events[4], "SEGMENT(UNH)");
220        assert_eq!(events[5], "SEGMENT(BGM)");
221        assert_eq!(events[6], "MESSAGE_END");
222        assert_eq!(events[7], "SEGMENT(UNT)");
223        assert_eq!(events[8], "INTERCHANGE_END");
224        assert_eq!(events[9], "SEGMENT(UNZ)");
225    }
226
227    #[test]
228    fn test_parse_without_una() {
229        let input = b"UNB+UNOC:3+SENDER+RECEIVER'UNZ+0+REF'";
230
231        let mut handler = EventCollector::new();
232        EdifactStreamParser::parse(input, &mut handler).unwrap();
233
234        let events = handler.events();
235        assert_eq!(events[0], "DELIMITERS(una=false)");
236        assert_eq!(events[1], "INTERCHANGE_START(UNB)");
237    }
238
239    #[test]
240    fn test_parse_handler_stops_early() {
241        struct StopOnBgm {
242            segments_seen: Vec<String>,
243        }
244        impl EdifactHandler for StopOnBgm {
245            fn on_segment(&mut self, seg: &RawSegment) -> Control {
246                self.segments_seen.push(seg.id.to_string());
247                if seg.is("BGM") {
248                    Control::Stop
249                } else {
250                    Control::Continue
251                }
252            }
253        }
254
255        let input = b"UNA:+.? 'UNB+UNOC:3'UNH+001'BGM+E03'DTM+137:20250101'UNT+3+001'UNZ+1'";
256        let mut handler = StopOnBgm {
257            segments_seen: Vec::new(),
258        };
259        EdifactStreamParser::parse(input, &mut handler).unwrap();
260
261        // Should have seen UNB, UNH, BGM but NOT DTM, UNT, UNZ
262        assert_eq!(handler.segments_seen, vec!["UNB", "UNH", "BGM"]);
263    }
264
265    #[test]
266    fn test_parse_message_numbering() {
267        struct PositionTracker {
268            positions: Vec<(String, u32)>,
269        }
270        impl EdifactHandler for PositionTracker {
271            fn on_segment(&mut self, seg: &RawSegment) -> Control {
272                self.positions
273                    .push((seg.id.to_string(), seg.position.message_number));
274                Control::Continue
275            }
276        }
277
278        let input =
279            b"UNA:+.? 'UNB+UNOC:3'UNH+001'BGM+E03'UNT+2+001'UNH+002'BGM+E03'UNT+2+002'UNZ+2'";
280        let mut handler = PositionTracker {
281            positions: Vec::new(),
282        };
283        EdifactStreamParser::parse(input, &mut handler).unwrap();
284
285        // UNB is outside messages (message_number=0)
286        assert_eq!(handler.positions[0], ("UNB".to_string(), 0));
287        // First message
288        assert_eq!(handler.positions[1], ("UNH".to_string(), 1));
289        assert_eq!(handler.positions[2], ("BGM".to_string(), 1));
290        assert_eq!(handler.positions[3], ("UNT".to_string(), 1));
291        // Second message
292        assert_eq!(handler.positions[4], ("UNH".to_string(), 2));
293        assert_eq!(handler.positions[5], ("BGM".to_string(), 2));
294        assert_eq!(handler.positions[6], ("UNT".to_string(), 2));
295        // UNZ is outside messages
296        assert_eq!(handler.positions[7], ("UNZ".to_string(), 0));
297    }
298
299    #[test]
300    fn test_parse_empty_input() {
301        struct NoOp;
302        impl EdifactHandler for NoOp {}
303
304        let mut handler = NoOp;
305        let result = EdifactStreamParser::parse(b"", &mut handler);
306        assert!(result.is_ok());
307    }
308
309    #[test]
310    fn test_parse_iso_8859_1_preserves_characters() {
311        // EDIFACT with ISO-8859-1 encoded German characters:
312        // ß = 0xDF, ö = 0xF6, ü = 0xFC in ISO-8859-1
313        // These are single bytes > 0x7F, NOT valid UTF-8.
314        let input: Vec<u8> = [
315            b"UNA:+.? '".as_slice(),
316            b"UNB+UNOC:3+SENDER+RECEIVER+210101:1200+REF'",
317            b"UNH+001+UTILMD:D:11A:UN'",
318            // NAD segment with "Müller" where ü = 0xFC (ISO-8859-1)
319            b"NAD+Z09+++M",
320            &[0xFC], // ü in ISO-8859-1
321            b"ller:Max::::Herr'",
322            // LOC segment with "Straße" where ß = 0xDF (ISO-8859-1)
323            b"LOC+Z16+++Hauptstra",
324            &[0xDF], // ß in ISO-8859-1
325            b"e::5'",
326            b"UNT+4+001'",
327            b"UNZ+1+REF'",
328        ]
329        .concat();
330
331        struct SegCollector {
332            segments: Vec<(String, Vec<Vec<String>>)>,
333        }
334        impl EdifactHandler for SegCollector {
335            fn on_segment(&mut self, seg: &RawSegment) -> Control {
336                self.segments.push((
337                    seg.id.to_string(),
338                    seg.elements
339                        .iter()
340                        .map(|e| e.iter().map(|c| c.to_string()).collect())
341                        .collect(),
342                ));
343                Control::Continue
344            }
345        }
346
347        let mut handler = SegCollector {
348            segments: Vec::new(),
349        };
350        EdifactStreamParser::parse(&input, &mut handler).unwrap();
351
352        // Find NAD segment — should contain "Müller" (ü transcoded to UTF-8)
353        // NAD+Z09+++Müller:Max::::Herr → elements: [Z09], [], [], [Müller,Max,...,Herr]
354        let nad = handler
355            .segments
356            .iter()
357            .find(|(id, _)| id == "NAD")
358            .expect("NAD segment should be present");
359        let name = &nad.1[3][0]; // element 3, component 0
360        assert!(
361            name.contains("ller"),
362            "NAD name component should contain 'ller', got: {:?}",
363            name
364        );
365        assert!(
366            name.contains('ü'),
367            "NAD name should contain ü (transcoded from ISO-8859-1 0xFC), got: {:?}",
368            name
369        );
370
371        // Find LOC segment — should contain "Straße" (ß transcoded to UTF-8)
372        // LOC+Z16+++Hauptstraße::5 → elements: [Z16], [], [], [Hauptstraße,,5]
373        let loc = handler
374            .segments
375            .iter()
376            .find(|(id, _)| id == "LOC")
377            .expect("LOC segment should be present");
378        let street = &loc.1[3][0]; // element 3, component 0
379        assert!(
380            street.contains("stra"),
381            "LOC street should contain 'stra', got: {:?}",
382            street
383        );
384        assert!(
385            street.contains('ß'),
386            "LOC street should contain ß (transcoded from ISO-8859-1 0xDF), got: {:?}",
387            street
388        );
389    }
390
391    #[test]
392    fn test_parse_real_world_dtm_with_timezone() {
393        struct DtmCollector {
394            dtm_values: Vec<String>,
395        }
396        impl EdifactHandler for DtmCollector {
397            fn on_segment(&mut self, seg: &RawSegment) -> Control {
398                if seg.is("DTM") {
399                    let qualifier = seg.get_component(0, 0);
400                    let value = seg.get_component(0, 1);
401                    self.dtm_values.push(format!("{}={}", qualifier, value));
402                }
403                Control::Continue
404            }
405        }
406
407        let input = b"UNA:+.? 'UNB+UNOC:3'UNH+001'DTM+137:202506190130?+00:303'UNT+2+001'UNZ+1'";
408        let mut handler = DtmCollector {
409            dtm_values: Vec::new(),
410        };
411        EdifactStreamParser::parse(input, &mut handler).unwrap();
412
413        assert_eq!(handler.dtm_values.len(), 1);
414        assert_eq!(handler.dtm_values[0], "137=202506190130?+00");
415    }
416
417    mod fuzz {
418        use super::*;
419        use proptest::prelude::*;
420
421        /// A handler that does nothing but exercises all callbacks.
422        struct FuzzHandler {
423            segment_count: usize,
424        }
425
426        impl EdifactHandler for FuzzHandler {
427            fn on_delimiters(&mut self, _d: &EdifactDelimiters, _una: bool) {}
428
429            fn on_interchange_start(&mut self, _unb: &RawSegment) -> Control {
430                Control::Continue
431            }
432
433            fn on_message_start(&mut self, _unh: &RawSegment) -> Control {
434                Control::Continue
435            }
436
437            fn on_segment(&mut self, _seg: &RawSegment) -> Control {
438                self.segment_count += 1;
439                if self.segment_count > 10_000 {
440                    Control::Stop // safety valve for huge inputs
441                } else {
442                    Control::Continue
443                }
444            }
445
446            fn on_message_end(&mut self, _unt: &RawSegment) {}
447            fn on_interchange_end(&mut self, _unz: &RawSegment) {}
448
449            fn on_error(&mut self, _error: ParseError) -> Control {
450                Control::Continue // try to keep going
451            }
452        }
453
454        proptest! {
455            #[test]
456            fn parser_never_panics_on_arbitrary_input(input in proptest::collection::vec(any::<u8>(), 0..1024)) {
457                let mut handler = FuzzHandler { segment_count: 0 };
458                // Must not panic — errors are OK, panics are NOT
459                let _ = EdifactStreamParser::parse(&input, &mut handler);
460            }
461
462            #[test]
463            fn parser_never_panics_on_ascii_input(input in "[A-Z0-9:+.?' \n\r]{0,512}") {
464                let mut handler = FuzzHandler { segment_count: 0 };
465                let _ = EdifactStreamParser::parse(input.as_bytes(), &mut handler);
466            }
467
468            #[test]
469            fn parser_handles_valid_looking_messages(
470                sender in "[A-Z0-9]{10,13}",
471                receiver in "[A-Z0-9]{10,13}",
472                ref_num in "[A-Z0-9]{5,10}",
473            ) {
474                let msg = format!(
475                    "UNA:+.? 'UNB+UNOC:3+{}+{}+210101:1200+{}'UNZ+0+{}'",
476                    sender, receiver, ref_num, ref_num,
477                );
478                let mut handler = FuzzHandler { segment_count: 0 };
479                let result = EdifactStreamParser::parse(msg.as_bytes(), &mut handler);
480                prop_assert!(result.is_ok());
481                prop_assert!(handler.segment_count >= 2); // at least UNB and UNZ
482            }
483        }
484    }
485}
edifact_parser/parser.rs

edifact_parser/
parser.rs
