automapper_validation/expr/
token.rs

1//! Tokenizer for condition expression strings.
2
3use crate::error::ParseError;
4
5/// Token types produced by the condition expression tokenizer.
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub enum Token {
8    /// A condition reference number, e.g., `931` from `[931]`.
9    ConditionId(String),
10    /// AND operator (`∧` or `AND`).
11    And,
12    /// OR operator (`∨` or `OR`).
13    Or,
14    /// XOR operator (`⊻` or `XOR`).
15    Xor,
16    /// NOT operator (`NOT`).
17    Not,
18    /// Opening parenthesis `(`.
19    LeftParen,
20    /// Closing parenthesis `)`.
21    RightParen,
22}
23
24/// A token with its position in the source string.
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct SpannedToken {
27    pub token: Token,
28    pub position: usize,
29}
30
31/// AHB status prefixes that are stripped before tokenizing.
32const STATUS_PREFIXES: &[&str] = &["Muss", "Soll", "Kann", "X"];
33
34/// Strip the AHB status prefix (Muss, Soll, Kann, X) from the input.
35///
36/// Returns the remainder of the string after the prefix, or the original
37/// string if no prefix is found.
38pub fn strip_status_prefix(input: &str) -> &str {
39    let trimmed = input.trim();
40    for prefix in STATUS_PREFIXES {
41        if let Some(rest) = trimmed.strip_prefix(prefix) {
42            let rest = rest.trim_start();
43            if !rest.is_empty() {
44                return rest;
45            }
46        }
47    }
48    trimmed
49}
50
51/// Tokenize an AHB condition expression string.
52///
53/// The input should already have the status prefix stripped.
54pub fn tokenize(input: &str) -> Result<Vec<SpannedToken>, ParseError> {
55    let mut tokens = Vec::new();
56    let chars: Vec<char> = input.chars().collect();
57    let mut i = 0;
58
59    while i < chars.len() {
60        let c = chars[i];
61
62        // Skip whitespace
63        if c.is_whitespace() {
64            i += 1;
65            continue;
66        }
67
68        let position = i;
69
70        // Parentheses
71        if c == '(' {
72            tokens.push(SpannedToken {
73                token: Token::LeftParen,
74                position,
75            });
76            i += 1;
77            continue;
78        }
79        if c == ')' {
80            tokens.push(SpannedToken {
81                token: Token::RightParen,
82                position,
83            });
84            i += 1;
85            continue;
86        }
87
88        // Unicode operators
89        if c == '\u{2227}' {
90            // ∧ AND
91            tokens.push(SpannedToken {
92                token: Token::And,
93                position,
94            });
95            i += 1;
96            continue;
97        }
98        if c == '\u{2228}' {
99            // ∨ OR
100            tokens.push(SpannedToken {
101                token: Token::Or,
102                position,
103            });
104            i += 1;
105            continue;
106        }
107        if c == '\u{22BB}' {
108            // ⊻ XOR
109            tokens.push(SpannedToken {
110                token: Token::Xor,
111                position,
112            });
113            i += 1;
114            continue;
115        }
116
117        // Condition reference [...]
118        if c == '[' {
119            let start = i;
120            i += 1;
121            while i < chars.len() && chars[i] != ']' {
122                i += 1;
123            }
124            if i < chars.len() {
125                let content: String = chars[start + 1..i].iter().collect();
126                tokens.push(SpannedToken {
127                    token: Token::ConditionId(content),
128                    position: start,
129                });
130                i += 1; // skip closing ]
131            } else {
132                let content: String = chars[start + 1..].iter().collect();
133                return Err(ParseError::InvalidConditionRef { content });
134            }
135            continue;
136        }
137
138        // Text keywords: AND, OR, XOR, NOT (case-insensitive)
139        if c.is_ascii_alphabetic() {
140            let start = i;
141            while i < chars.len() && chars[i].is_ascii_alphabetic() {
142                i += 1;
143            }
144            let word: String = chars[start..i].iter().collect();
145            match word.to_uppercase().as_str() {
146                "AND" => tokens.push(SpannedToken {
147                    token: Token::And,
148                    position: start,
149                }),
150                // A capital `V` is how some AHB editions spell `∨`.
151                "OR" | "V" if word != "v" => tokens.push(SpannedToken {
152                    token: Token::Or,
153                    position: start,
154                }),
155                "XOR" => tokens.push(SpannedToken {
156                    token: Token::Xor,
157                    position: start,
158                }),
159                "NOT" => tokens.push(SpannedToken {
160                    token: Token::Not,
161                    position: start,
162                }),
163                _ => {
164                    // Skip unknown words (could be status prefix remnants)
165                }
166            }
167            continue;
168        }
169
170        // Skip unknown characters
171        i += 1;
172    }
173
174    Ok(tokens)
175}
176
177#[cfg(test)]
178mod tests {
179    use super::*;
180
181    /// The AHB writes some ORs as a capital `V` instead of `∨` — INVOIC
182    /// 31001–31011 LOC 3225: `X ([950] [509] ∧ ([64] V [70])) V ([960] …)`.
183    /// Dropped as an unknown word, it turned the OR into an implicit AND, so
184    /// no location ID could ever satisfy the rule.
185    #[test]
186    fn a_capital_v_between_operands_is_an_or() {
187        let kinds = |s: &str| {
188            tokenize(s)
189                .unwrap()
190                .into_iter()
191                .map(|t| t.token)
192                .collect::<Vec<_>>()
193        };
194        assert_eq!(kinds("[64] V [70]"), kinds("[64] ∨ [70]"));
195        assert_eq!(kinds("([950]) V ([960])"), kinds("([950]) ∨ ([960])"));
196    }
197
198    // --- strip_status_prefix tests ---
199
200    #[test]
201    fn test_strip_muss_prefix() {
202        assert_eq!(strip_status_prefix("Muss [494]"), "[494]");
203    }
204
205    #[test]
206    fn test_strip_soll_prefix() {
207        assert_eq!(strip_status_prefix("Soll [494]"), "[494]");
208    }
209
210    #[test]
211    fn test_strip_kann_prefix() {
212        assert_eq!(strip_status_prefix("Kann [182] ∧ [6]"), "[182] ∧ [6]");
213    }
214
215    #[test]
216    fn test_strip_x_prefix() {
217        assert_eq!(
218            strip_status_prefix("X (([939][14]) ∨ ([940][15]))"),
219            "(([939][14]) ∨ ([940][15]))"
220        );
221    }
222
223    #[test]
224    fn test_strip_no_prefix() {
225        assert_eq!(strip_status_prefix("[1] ∧ [2]"), "[1] ∧ [2]");
226    }
227
228    #[test]
229    fn test_strip_muss_only_returns_trimmed() {
230        // "Muss" alone with nothing after has no conditions
231        assert_eq!(strip_status_prefix("Muss"), "Muss");
232    }
233
234    #[test]
235    fn test_strip_whitespace_only() {
236        assert_eq!(strip_status_prefix("   "), "");
237    }
238
239    #[test]
240    fn test_strip_preserves_leading_whitespace_in_content() {
241        assert_eq!(strip_status_prefix("Muss   [1]"), "[1]");
242    }
243
244    // --- tokenize tests ---
245
246    #[test]
247    fn test_tokenize_single_condition() {
248        let tokens = tokenize("[931]").unwrap();
249        assert_eq!(tokens.len(), 1);
250        assert_eq!(tokens[0].token, Token::ConditionId("931".to_string()));
251    }
252
253    #[test]
254    fn test_tokenize_and_unicode() {
255        let tokens = tokenize("[1] ∧ [2]").unwrap();
256        assert_eq!(tokens.len(), 3);
257        assert_eq!(tokens[0].token, Token::ConditionId("1".to_string()));
258        assert_eq!(tokens[1].token, Token::And);
259        assert_eq!(tokens[2].token, Token::ConditionId("2".to_string()));
260    }
261
262    #[test]
263    fn test_tokenize_or_unicode() {
264        let tokens = tokenize("[1] ∨ [2]").unwrap();
265        assert_eq!(tokens[1].token, Token::Or);
266    }
267
268    #[test]
269    fn test_tokenize_xor_unicode() {
270        let tokens = tokenize("[1] ⊻ [2]").unwrap();
271        assert_eq!(tokens[1].token, Token::Xor);
272    }
273
274    #[test]
275    fn test_tokenize_text_keywords() {
276        let tokens = tokenize("[1] AND [2] OR [3] XOR [4]").unwrap();
277        assert_eq!(tokens.len(), 7);
278        assert_eq!(tokens[1].token, Token::And);
279        assert_eq!(tokens[3].token, Token::Or);
280        assert_eq!(tokens[5].token, Token::Xor);
281    }
282
283    #[test]
284    fn test_tokenize_not_keyword() {
285        let tokens = tokenize("NOT [1]").unwrap();
286        assert_eq!(tokens.len(), 2);
287        assert_eq!(tokens[0].token, Token::Not);
288        assert_eq!(tokens[1].token, Token::ConditionId("1".to_string()));
289    }
290
291    #[test]
292    fn test_tokenize_parentheses() {
293        let tokens = tokenize("([1] ∨ [2]) ∧ [3]").unwrap();
294        assert_eq!(tokens.len(), 7);
295        assert_eq!(tokens[0].token, Token::LeftParen);
296        assert_eq!(tokens[4].token, Token::RightParen);
297    }
298
299    #[test]
300    fn test_tokenize_adjacent_conditions_no_space() {
301        let tokens = tokenize("[939][14]").unwrap();
302        assert_eq!(tokens.len(), 2);
303        assert_eq!(tokens[0].token, Token::ConditionId("939".to_string()));
304        assert_eq!(tokens[1].token, Token::ConditionId("14".to_string()));
305    }
306
307    #[test]
308    fn test_tokenize_package_condition() {
309        let tokens = tokenize("[10P1..5]").unwrap();
310        assert_eq!(tokens.len(), 1);
311        assert_eq!(tokens[0].token, Token::ConditionId("10P1..5".to_string()));
312    }
313
314    #[test]
315    fn test_tokenize_time_condition() {
316        let tokens = tokenize("[UB1]").unwrap();
317        assert_eq!(tokens.len(), 1);
318        assert_eq!(tokens[0].token, Token::ConditionId("UB1".to_string()));
319    }
320
321    #[test]
322    fn test_tokenize_tabs_and_multiple_spaces() {
323        let tokens = tokenize("[1]\t∧\t[2]").unwrap();
324        assert_eq!(tokens.len(), 3);
325        assert_eq!(tokens[1].token, Token::And);
326    }
327
328    #[test]
329    fn test_tokenize_multiple_spaces() {
330        let tokens = tokenize("[1]    ∧    [2]").unwrap();
331        assert_eq!(tokens.len(), 3);
332    }
333
334    #[test]
335    fn test_tokenize_empty_string() {
336        let tokens = tokenize("").unwrap();
337        assert!(tokens.is_empty());
338    }
339
340    #[test]
341    fn test_tokenize_complex_real_world() {
342        // "X (([939] [147]) ∨ ([940] [148])) ∧ [567]"
343        // After prefix strip: "(([939] [147]) ∨ ([940] [148])) ∧ [567]"
344        let tokens = tokenize("(([939] [147]) ∨ ([940] [148])) ∧ [567]").unwrap();
345        assert_eq!(tokens.len(), 13);
346        assert_eq!(tokens[0].token, Token::LeftParen);
347        assert_eq!(tokens[1].token, Token::LeftParen);
348        assert_eq!(tokens[2].token, Token::ConditionId("939".to_string()));
349        assert_eq!(tokens[3].token, Token::ConditionId("147".to_string()));
350        assert_eq!(tokens[4].token, Token::RightParen);
351        assert_eq!(tokens[5].token, Token::Or);
352        assert_eq!(tokens[6].token, Token::LeftParen);
353        assert_eq!(tokens[7].token, Token::ConditionId("940".to_string()));
354        assert_eq!(tokens[8].token, Token::ConditionId("148".to_string()));
355        assert_eq!(tokens[9].token, Token::RightParen);
356        assert_eq!(tokens[10].token, Token::RightParen);
357        assert_eq!(tokens[11].token, Token::And);
358        assert_eq!(tokens[12].token, Token::ConditionId("567".to_string()));
359    }
360
361    #[test]
362    fn test_tokenize_positions_are_correct() {
363        let tokens = tokenize("[1] ∧ [2]").unwrap();
364        assert_eq!(tokens[0].position, 0); // [
365        assert_eq!(tokens[2].position, 6); // [ of [2] (∧ is a single char in char index)
366    }
367
368    #[test]
369    fn test_tokenize_case_insensitive_keywords() {
370        let tokens = tokenize("[1] and [2] or [3]").unwrap();
371        assert_eq!(tokens[1].token, Token::And);
372        assert_eq!(tokens[3].token, Token::Or);
373    }
374
375    #[test]
376    fn test_tokenize_unclosed_bracket_returns_error() {
377        let result = tokenize("[931");
378        assert!(result.is_err());
379    }
380}
automapper_validation/expr/token.rs

automapper_validation/expr/
token.rs
