sieve/compiler/lexer/
string.rs

1/*
2 * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
3 *
4 * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
5 */
6
7use crate::{
8    Envelope, MAX_MATCH_VARIABLES,
9    compiler::{
10        ContentTypePart, ErrorType, HeaderPart, HeaderVariable, MessagePart, Number,
11        ReceivedHostname, ReceivedPart, Value, VariableType,
12        grammar::{
13            AddressPart,
14            expr::{self},
15            instruction::CompilerState,
16        },
17    },
18};
19use mail_parser::HeaderName;
20use std::fmt::Display;
21
22enum State {
23    None,
24    Variable,
25    Encoded {
26        is_unicode: bool,
27        initial_buf_size: usize,
28    },
29}
30
31impl CompilerState<'_> {
32    pub(crate) fn tokenize_string(
33        &mut self,
34        bytes: &[u8],
35        parse_decoded: bool,
36    ) -> Result<Value, ErrorType> {
37        let mut state = State::None;
38        let mut items = Vec::with_capacity(3);
39        let mut last_ch = 0;
40
41        let mut var_start_pos = usize::MAX;
42        let mut var_is_number = true;
43        let mut var_has_namespace = false;
44
45        let mut text_has_digits = true;
46        let mut text_has_dots = false;
47
48        let mut hex_start = usize::MAX;
49        let mut decode_buf = Vec::with_capacity(bytes.len());
50
51        for (pos, &ch) in bytes.iter().enumerate() {
52            let mut is_var_error = false;
53
54            match state {
55                State::None => match ch {
56                    b'{' if last_ch == b'$' => {
57                        decode_buf.pop();
58                        var_start_pos = pos + 1;
59                        var_is_number = true;
60                        var_has_namespace = false;
61                        state = State::Variable;
62                    }
63                    b'.' => {
64                        if text_has_dots {
65                            text_has_digits = false;
66                        } else {
67                            text_has_dots = true;
68                        }
69                        decode_buf.push(ch);
70                    }
71                    b'0'..=b'9' => {
72                        decode_buf.push(ch);
73                    }
74                    _ => {
75                        text_has_digits = false;
76                        decode_buf.push(ch);
77                    }
78                },
79                State::Variable => match ch {
80                    b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'[' | b']' | b'*' | b'-' => {
81                        var_is_number = false;
82                    }
83                    b'.' => {
84                        var_is_number = false;
85                        var_has_namespace = true;
86                    }
87                    b'0'..=b'9' => {}
88                    b'}' if pos > var_start_pos => {
89                        // Add any text before the variable
90                        if !decode_buf.is_empty() {
91                            self.add_value(
92                                &mut items,
93                                &decode_buf,
94                                parse_decoded,
95                                text_has_digits,
96                                text_has_dots,
97                            )?;
98                            decode_buf.clear();
99                            text_has_digits = true;
100                            text_has_dots = false;
101                        }
102
103                        // Parse variable type
104                        let var_name = std::str::from_utf8(&bytes[var_start_pos..pos]).unwrap();
105                        let var_type = if !var_is_number {
106                            self.parse_variable(var_name, var_has_namespace)
107                        } else {
108                            self.parse_match_variable(var_name)
109                        };
110
111                        match var_type {
112                            Ok(Some(var)) => items.push(Value::Variable(var)),
113                            Ok(None) => {}
114                            Err(ErrorType::InvalidNamespace(_) | ErrorType::InvalidEnvelope(_)) => {
115                                is_var_error = true;
116                            }
117                            Err(e) => return Err(e),
118                        }
119
120                        state = State::None;
121                    }
122                    b':' => {
123                        if parse_decoded && !var_has_namespace {
124                            match bytes.get(var_start_pos..pos) {
125                                Some(enc) if enc.eq_ignore_ascii_case(b"hex") => {
126                                    state = State::Encoded {
127                                        is_unicode: false,
128                                        initial_buf_size: decode_buf.len(),
129                                    };
130                                }
131                                Some(enc) if enc.eq_ignore_ascii_case(b"unicode") => {
132                                    state = State::Encoded {
133                                        is_unicode: true,
134                                        initial_buf_size: decode_buf.len(),
135                                    };
136                                }
137                                _ => {
138                                    is_var_error = true;
139                                }
140                            }
141                        } else if var_has_namespace {
142                            var_is_number = false;
143                        } else {
144                            is_var_error = true;
145                        }
146                    }
147                    _ => {
148                        is_var_error = true;
149                    }
150                },
151
152                State::Encoded {
153                    is_unicode,
154                    initial_buf_size,
155                } => match ch {
156                    b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => {
157                        if hex_start == usize::MAX {
158                            hex_start = pos;
159                        }
160                    }
161                    b' ' | b'\t' | b'\r' | b'\n' | b'}' => {
162                        if hex_start != usize::MAX {
163                            let code = std::str::from_utf8(&bytes[hex_start..pos]).unwrap();
164                            hex_start = usize::MAX;
165
166                            if !is_unicode {
167                                if let Ok(ch) = u8::from_str_radix(code, 16) {
168                                    decode_buf.push(ch);
169                                } else {
170                                    is_var_error = true;
171                                }
172                            } else if let Ok(ch) = u32::from_str_radix(code, 16) {
173                                let mut buf = [0; 4];
174                                decode_buf.extend_from_slice(
175                                    char::from_u32(ch)
176                                        .ok_or(ErrorType::InvalidUnicodeSequence(ch))?
177                                        .encode_utf8(&mut buf)
178                                        .as_bytes(),
179                                );
180                            } else {
181                                is_var_error = true;
182                            }
183                        }
184                        if ch == b'}' {
185                            if decode_buf.len() != initial_buf_size {
186                                state = State::None;
187                            } else {
188                                is_var_error = true;
189                            }
190                        }
191                    }
192                    _ => {
193                        is_var_error = true;
194                    }
195                },
196            }
197
198            if is_var_error {
199                if let State::Encoded {
200                    initial_buf_size, ..
201                } = state
202                    && initial_buf_size != decode_buf.len()
203                {
204                    decode_buf.truncate(initial_buf_size);
205                }
206                decode_buf.extend_from_slice(&bytes[var_start_pos - 2..pos + 1]);
207                hex_start = usize::MAX;
208                state = State::None;
209            }
210
211            last_ch = ch;
212        }
213
214        match state {
215            State::Variable => {
216                decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]);
217            }
218            State::Encoded {
219                initial_buf_size, ..
220            } => {
221                if initial_buf_size != decode_buf.len() {
222                    decode_buf.truncate(initial_buf_size);
223                }
224                decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]);
225            }
226            State::None => (),
227        }
228
229        if !decode_buf.is_empty() {
230            self.add_value(
231                &mut items,
232                &decode_buf,
233                parse_decoded,
234                text_has_digits,
235                text_has_dots,
236            )?;
237        }
238
239        Ok(match items.len() {
240            1 => items.pop().unwrap(),
241            0 => self.text(""),
242            _ => Value::List(items.into()),
243        })
244    }
245
246    fn parse_match_variable(&mut self, var_name: &str) -> Result<Option<VariableType>, ErrorType> {
247        let num = var_name
248            .parse()
249            .map_err(|_| ErrorType::InvalidNumber(var_name.to_string()))?;
250        if num < MAX_MATCH_VARIABLES as usize {
251            if self.register_match_var(num) {
252                let total_vars = num + 1;
253                if total_vars > self.vars_match_max {
254                    self.vars_match_max = total_vars;
255                }
256                Ok(Some(VariableType::Match(num as u8)))
257            } else {
258                Ok(None)
259            }
260        } else {
261            Err(ErrorType::InvalidMatchVariable(num))
262        }
263    }
264
265    pub fn parse_variable(
266        &self,
267        var_name: &str,
268        maybe_namespace: bool,
269    ) -> Result<Option<VariableType>, ErrorType> {
270        if !maybe_namespace {
271            if self.is_var_global(var_name) {
272                Ok(Some(VariableType::Global(var_name.to_string())))
273            } else if let Some(var_id) = self.get_local_var(var_name) {
274                Ok(Some(VariableType::Local(var_id)))
275            } else {
276                Ok(None)
277            }
278        } else {
279            let lowercase_name = if var_name.is_ascii() {
280                super::super::grammar::instruction::lowercase(var_name)
281            } else {
282                std::borrow::Cow::Owned(var_name.to_lowercase())
283            };
284            let var = if let Some((namespace, name)) = lowercase_name.split_once('.') {
285                if name.is_empty() {
286                    return Err(ErrorType::InvalidNamespace(var_name.to_string()));
287                }
288
289                let mut var = None;
290                hashify::fnc_map!(namespace.as_bytes(),
291                    "global" => {
292                        var = VariableType::Global(name.to_string()).into();
293                    },
294                    "t" => {
295                        var = VariableType::Global(name.to_string()).into();
296                    },
297                    "env" => {
298                        var = VariableType::Environment(name.to_string()).into();
299                    },
300                    "envelope" => {
301                        var = VariableType::Envelope(
302                            lookup_envelope(name)
303                                .ok_or_else(|| ErrorType::InvalidEnvelope(name.to_string()))?,
304                        )
305                        .into();
306                    },
307                    "header" => {
308                        var = self.parse_header_variable(name)?.into();
309                    },
310                    "body" => {
311                        var = VariableType::Part(
312                            lookup_body_part(name)
313                                .ok_or_else(|| ErrorType::InvalidNamespace(name.to_string()))?,
314                        )
315                        .into();
316                    },
317                    "part" => {
318                        var = VariableType::Part(
319                            lookup_message_part(name)
320                                .ok_or_else(|| ErrorType::InvalidNamespace(name.to_string()))?,
321                        )
322                        .into();
323                    },
324                    _ => {}
325                );
326
327                var.ok_or_else(|| ErrorType::InvalidNamespace(var_name.to_string()))?
328            } else if self.is_var_global(var_name) {
329                VariableType::Global(var_name.to_string())
330            } else if let Some(var_id) = self.get_local_var(var_name) {
331                VariableType::Local(var_id)
332            } else {
333                return Ok(None);
334            };
335
336            Ok(Some(var))
337        }
338    }
339
340    fn parse_header_variable(&self, var_name: &str) -> Result<VariableType, ErrorType> {
341        #[derive(Debug)]
342        enum State {
343            Name,
344            Index,
345            Part,
346            PartIndex,
347        }
348        let mut name = vec![];
349        let mut has_name = false;
350        let mut has_wildcard = false;
351        let mut hdr_name = String::new();
352        let mut hdr_index = String::new();
353        let mut part = String::new();
354        let mut part_index = String::new();
355        let mut state = State::Name;
356
357        for ch in var_name.chars() {
358            match state {
359                State::Name => match ch {
360                    '[' => {
361                        state = if hdr_index.is_empty() {
362                            State::Index
363                        } else if part.is_empty() {
364                            State::PartIndex
365                        } else {
366                            return Err(ErrorType::InvalidExpression(var_name.to_string()));
367                        };
368                        has_name = true;
369                    }
370                    '.' => {
371                        state = State::Part;
372                        has_name = true;
373                    }
374                    ' ' | '\t' | '\r' | '\n' => {}
375                    '*' if !has_wildcard && hdr_name.is_empty() && name.is_empty() => {
376                        has_wildcard = true;
377                    }
378                    ':' if !hdr_name.is_empty() && !has_wildcard => {
379                        name.push(
380                            HeaderName::parse(std::mem::take(&mut hdr_name)).ok_or_else(|| {
381                                ErrorType::InvalidExpression(var_name.to_string())
382                            })?,
383                        );
384                    }
385                    _ if !has_name && !has_wildcard => {
386                        hdr_name.push(ch);
387                    }
388                    _ => {
389                        return Err(ErrorType::InvalidExpression(var_name.to_string()));
390                    }
391                },
392                State::Index => match ch {
393                    ']' => {
394                        state = State::Name;
395                    }
396                    ' ' | '\t' | '\r' | '\n' => {}
397                    _ => {
398                        hdr_index.push(ch);
399                    }
400                },
401                State::Part => match ch {
402                    '[' => {
403                        state = State::PartIndex;
404                    }
405                    ' ' | '\t' | '\r' | '\n' => {}
406                    _ => {
407                        part.push(ch);
408                    }
409                },
410                State::PartIndex => match ch {
411                    ']' => {
412                        state = State::Name;
413                    }
414                    ' ' | '\t' | '\r' | '\n' => {}
415                    _ => {
416                        part_index.push(ch);
417                    }
418                },
419            }
420        }
421
422        if !hdr_name.is_empty() {
423            name.push(
424                HeaderName::parse(hdr_name)
425                    .ok_or_else(|| ErrorType::InvalidExpression(var_name.to_string()))?,
426            );
427        }
428
429        if !name.is_empty() || has_wildcard {
430            Ok(VariableType::Header(HeaderVariable {
431                name: name.into(),
432                part: HeaderPart::try_from(part.as_str())
433                    .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
434                index_hdr: match hdr_index.as_str() {
435                    "" => {
436                        if !has_wildcard {
437                            -1
438                        } else {
439                            0
440                        }
441                    }
442                    "*" => 0,
443                    _ => hdr_index
444                        .parse()
445                        .map(|v| if v == 0 { 1 } else { v })
446                        .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
447                },
448                index_part: match part_index.as_str() {
449                    "" => {
450                        if !has_wildcard {
451                            -1
452                        } else {
453                            0
454                        }
455                    }
456                    "*" => 0,
457                    _ => part_index
458                        .parse()
459                        .map(|v| if v == 0 { 1 } else { v })
460                        .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
461                },
462            }))
463        } else {
464            Err(ErrorType::InvalidExpression(var_name.to_string()))
465        }
466    }
467
468    pub fn parse_expr_fnc_or_var(
469        &self,
470        var_name: &str,
471        maybe_namespace: bool,
472    ) -> Result<expr::Token, String> {
473        match self.parse_variable(var_name, maybe_namespace) {
474            Ok(Some(var)) => Ok(expr::Token::Variable(var)),
475            _ => {
476                if let Some((id, num_args)) = self.compiler.functions.get(var_name) {
477                    Ok(expr::Token::Function {
478                        name: var_name.to_string(),
479                        id: *id,
480                        num_args: *num_args,
481                    })
482                } else {
483                    Err(format!("Invalid variable or function name {var_name:?}"))
484                }
485            }
486        }
487    }
488
489    #[inline(always)]
490    fn add_value(
491        &mut self,
492        items: &mut Vec<Value>,
493        buf: &[u8],
494        parse_decoded: bool,
495        has_digits: bool,
496        has_dots: bool,
497    ) -> Result<(), ErrorType> {
498        if !parse_decoded {
499            let value = if has_digits {
500                if has_dots {
501                    match std::str::from_utf8(buf)
502                        .ok()
503                        .and_then(|v| (v, v.parse::<f64>().ok()?).into())
504                    {
505                        Some((v, n)) if n.to_string() == v => Value::Number(Number::Float(n)),
506                        _ => self.text(String::from_utf8_lossy(buf)),
507                    }
508                } else {
509                    match std::str::from_utf8(buf)
510                        .ok()
511                        .and_then(|v| (v, v.parse::<i64>().ok()?).into())
512                    {
513                        Some((v, n)) if n.to_string() == v => Value::Number(Number::Integer(n)),
514                        _ => self.text(String::from_utf8_lossy(buf)),
515                    }
516                }
517            } else {
518                self.text(String::from_utf8_lossy(buf))
519            };
520            items.push(value);
521        } else {
522            match self.tokenize_string(buf, false)? {
523                Value::List(new_items) => items.extend(new_items),
524                item => items.push(item),
525            }
526        }
527
528        Ok(())
529    }
530}
531
532impl TryFrom<&str> for HeaderPart {
533    type Error = ();
534
535    fn try_from(value: &str) -> Result<Self, Self::Error> {
536        let (value, subvalue) = value.split_once('.').unwrap_or((value, ""));
537        if value.is_empty() {
538            return Ok(HeaderPart::Text);
539        }
540
541        let mut part = None;
542        hashify::fnc_map!(value.as_bytes(),
543            "text" => {
544                part = HeaderPart::Text.into();
545            },
546            "name" => {
547                part = HeaderPart::Address(AddressPart::Name).into();
548            },
549            "addr" => {
550                part = if !subvalue.is_empty() {
551                    HeaderPart::Address(AddressPart::try_from(subvalue)?)
552                } else {
553                    HeaderPart::Address(AddressPart::All)
554                }
555                .into();
556            },
557            "type" => {
558                part = HeaderPart::ContentType(ContentTypePart::Type).into();
559            },
560            "subtype" => {
561                part = HeaderPart::ContentType(ContentTypePart::Subtype).into();
562            },
563            "attr" => {
564                if !subvalue.is_empty() {
565                    part = HeaderPart::ContentType(ContentTypePart::Attribute(
566                        subvalue.to_string(),
567                    ))
568                    .into();
569                }
570            },
571            "rcvd" => {
572                part = if !subvalue.is_empty() {
573                    HeaderPart::Received(ReceivedPart::try_from(subvalue)?)
574                } else {
575                    HeaderPart::Text
576                }
577                .into();
578            },
579            "id" => {
580                part = HeaderPart::Id.into();
581            },
582            "raw" => {
583                part = HeaderPart::Raw.into();
584            },
585            "raw_name" => {
586                part = HeaderPart::RawName.into();
587            },
588            "date" => {
589                part = HeaderPart::Date.into();
590            },
591            "exists" => {
592                part = HeaderPart::Exists.into();
593            },
594            _ => {}
595        );
596
597        part.ok_or(())
598    }
599}
600
601impl TryFrom<&str> for ReceivedPart {
602    type Error = ();
603
604    fn try_from(value: &str) -> Result<Self, Self::Error> {
605        lookup_received_part(value).ok_or(())
606    }
607}
608
609impl TryFrom<&str> for AddressPart {
610    type Error = ();
611
612    fn try_from(value: &str) -> Result<Self, Self::Error> {
613        lookup_address_part(value).ok_or(())
614    }
615}
616
617fn lookup_envelope(input: &str) -> Option<Envelope> {
618    hashify::tiny_map!(
619        input.as_bytes(),
620        "from" => Envelope::From,
621        "to" => Envelope::To,
622        "by_time_absolute" => Envelope::ByTimeAbsolute,
623        "by_time_relative" => Envelope::ByTimeRelative,
624        "by_mode" => Envelope::ByMode,
625        "by_trace" => Envelope::ByTrace,
626        "notify" => Envelope::Notify,
627        "orcpt" => Envelope::Orcpt,
628        "ret" => Envelope::Ret,
629        "envid" => Envelope::Envid,
630    )
631}
632
633fn lookup_body_part(input: &str) -> Option<MessagePart> {
634    hashify::tiny_map!(
635        input.as_bytes(),
636        "text" => MessagePart::TextBody(false),
637        "html" => MessagePart::HtmlBody(false),
638        "to_text" => MessagePart::TextBody(true),
639        "to_html" => MessagePart::HtmlBody(true),
640    )
641}
642
643fn lookup_message_part(input: &str) -> Option<MessagePart> {
644    hashify::tiny_map!(
645        input.as_bytes(),
646        "text" => MessagePart::Contents,
647        "raw" => MessagePart::Raw,
648    )
649}
650
651fn lookup_received_part(input: &str) -> Option<ReceivedPart> {
652    hashify::tiny_map!(
653        input.as_bytes(),
654        "from" => ReceivedPart::From(ReceivedHostname::Any),
655        "from.name" => ReceivedPart::From(ReceivedHostname::Name),
656        "from.ip" => ReceivedPart::From(ReceivedHostname::Ip),
657        "ip" => ReceivedPart::FromIp,
658        "iprev" => ReceivedPart::FromIpRev,
659        "by" => ReceivedPart::By(ReceivedHostname::Any),
660        "by.name" => ReceivedPart::By(ReceivedHostname::Name),
661        "by.ip" => ReceivedPart::By(ReceivedHostname::Ip),
662        "for" => ReceivedPart::For,
663        "with" => ReceivedPart::With,
664        "tls" => ReceivedPart::TlsVersion,
665        "cipher" => ReceivedPart::TlsCipher,
666        "id" => ReceivedPart::Id,
667        "ident" => ReceivedPart::Ident,
668        "date" => ReceivedPart::Date,
669        "date.raw" => ReceivedPart::DateRaw,
670    )
671}
672
673fn lookup_address_part(input: &str) -> Option<AddressPart> {
674    hashify::tiny_map!(
675        input.as_bytes(),
676        "name" => AddressPart::Name,
677        "addr" => AddressPart::All,
678        "all" => AddressPart::All,
679        "addr.domain" => AddressPart::Domain,
680        "addr.local" => AddressPart::LocalPart,
681        "addr.user" => AddressPart::User,
682        "addr.detail" => AddressPart::Detail,
683    )
684}
685
686impl Display for VariableType {
687    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688        match self {
689            VariableType::Local(v) => write!(f, "${{{v}}}"),
690            VariableType::Match(v) => write!(f, "${{{v}}}"),
691            VariableType::Global(v) => write!(f, "${{global.{v}}}"),
692            VariableType::Environment(v) => write!(f, "${{env.{v}}}"),
693
694            VariableType::Envelope(env) => f.write_str(match env {
695                Envelope::From => "${{envelope.from}}",
696                Envelope::To => "${{envelope.to}}",
697                Envelope::ByTimeAbsolute => "${{envelope.by_time_absolute}}",
698                Envelope::ByTimeRelative => "${{envelope.by_time_relative}}",
699                Envelope::ByMode => "${{envelope.by_mode}}",
700                Envelope::ByTrace => "${{envelope.by_trace}}",
701                Envelope::Notify => "${{envelope.notify}}",
702                Envelope::Orcpt => "${{envelope.orcpt}}",
703                Envelope::Ret => "${{envelope.ret}}",
704                Envelope::Envid => "${{envelope.envit}}",
705            }),
706
707            VariableType::Header(hdr) => {
708                write!(
709                    f,
710                    "${{header.{}",
711                    hdr.name.first().map(|h| h.as_str()).unwrap_or_default()
712                )?;
713                if hdr.index_hdr != 0 {
714                    write!(f, "[{}]", hdr.index_hdr)?;
715                } else {
716                    f.write_str("[*]")?;
717                }
718                /*if hdr.part != HeaderPart::Text {
719                    f.write_str(".")?;
720                    f.write_str(match &hdr.part {
721                        HeaderPart::Name => "name",
722                        HeaderPart::Address => "address",
723                        HeaderPart::Type => "type",
724                        HeaderPart::Subtype => "subtype",
725                        HeaderPart::Raw => "raw",
726                        HeaderPart::Date => "date",
727                        HeaderPart::Attribute(attr) => attr.as_str(),
728                        HeaderPart::Text => unreachable!(),
729                    })?;
730                }*/
731                if hdr.index_part != 0 {
732                    write!(f, "[{}]", hdr.index_part)?;
733                } else {
734                    f.write_str("[*]")?;
735                }
736                f.write_str("}")
737            }
738            VariableType::Part(part) => {
739                write!(
740                    f,
741                    "${{{}",
742                    match part {
743                        MessagePart::TextBody(true) => "body.to_text",
744                        MessagePart::TextBody(false) => "body.text",
745                        MessagePart::HtmlBody(true) => "body.to_html",
746                        MessagePart::HtmlBody(false) => "body.html",
747                        MessagePart::Contents => "part.text",
748                        MessagePart::Raw => "part.raw",
749                    }
750                )?;
751                f.write_str("}")
752            }
753        }
754    }
755}
756
757#[cfg(test)]
758mod tests {
759
760    use mail_parser::HeaderName;
761
762    use super::Value;
763    use crate::compiler::grammar::instruction::{Block, CompilerState, Instruction, MAX_PARAMS};
764    use crate::compiler::grammar::test::Test;
765    use crate::compiler::grammar::tests::test_string::TestString;
766    use crate::compiler::grammar::{Comparator, MatchType};
767    use crate::compiler::lexer::tokenizer::Tokenizer;
768    use crate::compiler::lexer::word::Word;
769    use crate::compiler::{AddressPart, HeaderPart, HeaderVariable, VariableType};
770    use crate::{AHashSet, Compiler};
771
772    fn text(state: &mut CompilerState, value: &str) -> Value {
773        Value::Text(state.intern(value))
774    }
775
776    #[test]
777    fn tokenize_string() {
778        let c = Compiler::new();
779        let mut block = Block::new(Word::Not);
780        block.match_test_pos.push(0);
781        let mut compiler = CompilerState {
782            compiler: &c,
783            instructions: vec![Instruction::Test(Box::new(Test::String(Box::new(
784                TestString {
785                    match_type: MatchType::Regex(u64::MAX),
786                    comparator: Comparator::AsciiCaseMap,
787                    source: vec![Value::Variable(VariableType::Local(0))].into(),
788                    key_list: vec![Value::Variable(VariableType::Local(0))].into(),
789                    is_not: false,
790                },
791            ))))],
792            block_stack: Vec::new(),
793            block,
794            last_block_type: Word::Not,
795            vars_global: AHashSet::new(),
796            vars_num: 0,
797            vars_num_max: 0,
798            vars_local: 0,
799            tokens: Tokenizer::new(&c, b""),
800            vars_match_max: usize::MAX,
801            param_check: [false; MAX_PARAMS],
802            includes_num: 0,
803            constants: Vec::new(),
804            constants_map: hashbrown::HashTable::new(),
805            hasher: ahash::RandomState::new(),
806        };
807
808        for (input, expected_result) in [
809            ("$${hex:24 24}", text(&mut compiler, "$$$")),
810            ("$${hex:40}", text(&mut compiler, "$@")),
811            ("${hex: 40 }", text(&mut compiler, "@")),
812            ("${HEX: 40}", text(&mut compiler, "@")),
813            ("${hex:40", text(&mut compiler, "${hex:40")),
814            ("${hex:400}", text(&mut compiler, "${hex:400}")),
815            ("${hex:4${hex:30}}", text(&mut compiler, "${hex:40}")),
816            ("${unicode:40}", text(&mut compiler, "@")),
817            ("${ unicode:40}", text(&mut compiler, "${ unicode:40}")),
818            ("${UNICODE:40}", text(&mut compiler, "@")),
819            ("${UnICoDE:0000040}", text(&mut compiler, "@")),
820            ("${Unicode:40}", text(&mut compiler, "@")),
821            ("${Unicode:40 40 ", text(&mut compiler, "${Unicode:40 40 ")),
822            ("${Unicode:Cool}", text(&mut compiler, "${Unicode:Cool}")),
823            ("", text(&mut compiler, "")),
824            (
825                "${global.full}",
826                Value::Variable(VariableType::Global("full".to_string())),
827            ),
828            (
829                "${BAD${global.Company}",
830                Value::List(
831                    vec![
832                        text(&mut compiler, "${BAD"),
833                        Value::Variable(VariableType::Global("company".to_string())),
834                    ]
835                    .into(),
836                ),
837            ),
838            (
839                "${President, ${global.Company} Inc.}",
840                Value::List(
841                    vec![
842                        text(&mut compiler, "${President, "),
843                        Value::Variable(VariableType::Global("company".to_string())),
844                        text(&mut compiler, " Inc.}"),
845                    ]
846                    .into(),
847                ),
848            ),
849            (
850                "dear${hex:20 24 7b}global.Name}",
851                Value::List(
852                    vec![
853                        text(&mut compiler, "dear "),
854                        Value::Variable(VariableType::Global("name".to_string())),
855                    ]
856                    .into(),
857                ),
858            ),
859            (
860                "INBOX.lists.${2}",
861                Value::List(
862                    vec![
863                        text(&mut compiler, "INBOX.lists."),
864                        Value::Variable(VariableType::Match(2)),
865                    ]
866                    .into(),
867                ),
868            ),
869            (
870                "Ein unerh${unicode:00F6}rt gro${unicode:00DF}er Test",
871                text(&mut compiler, "Ein unerhört großer Test"),
872            ),
873            ("&%${}!", text(&mut compiler, "&%${}!")),
874            ("${doh!}", text(&mut compiler, "${doh!}")),
875            (
876                "${hex: 20 }${global.hi}${hex: 20 }",
877                Value::List(
878                    vec![
879                        text(&mut compiler, " "),
880                        Value::Variable(VariableType::Global("hi".to_string())),
881                        text(&mut compiler, " "),
882                    ]
883                    .into(),
884                ),
885            ),
886            (
887                "${hex:20 24 7b z}${global.hi}${unicode:}${unicode: }${hex:20}",
888                Value::List(
889                    vec![
890                        text(&mut compiler, "${hex:20 24 7b z}"),
891                        Value::Variable(VariableType::Global("hi".to_string())),
892                        text(&mut compiler, "${unicode:}${unicode: } "),
893                    ]
894                    .into(),
895                ),
896            ),
897            (
898                "${header.from}",
899                Value::Variable(VariableType::Header(HeaderVariable {
900                    name: vec![HeaderName::From].into(),
901                    part: HeaderPart::Text,
902                    index_hdr: -1,
903                    index_part: -1,
904                })),
905            ),
906            (
907                "${header.from.addr}",
908                Value::Variable(VariableType::Header(HeaderVariable {
909                    name: vec![HeaderName::From].into(),
910                    part: HeaderPart::Address(AddressPart::All),
911                    index_hdr: -1,
912                    index_part: -1,
913                })),
914            ),
915            (
916                "${header.from[1]}",
917                Value::Variable(VariableType::Header(HeaderVariable {
918                    name: vec![HeaderName::From].into(),
919                    part: HeaderPart::Text,
920                    index_hdr: 1,
921                    index_part: -1,
922                })),
923            ),
924            (
925                "${header.from[*]}",
926                Value::Variable(VariableType::Header(HeaderVariable {
927                    name: vec![HeaderName::From].into(),
928                    part: HeaderPart::Text,
929                    index_hdr: 0,
930                    index_part: -1,
931                })),
932            ),
933            (
934                "${header.from[20].name}",
935                Value::Variable(VariableType::Header(HeaderVariable {
936                    name: vec![HeaderName::From].into(),
937                    part: HeaderPart::Address(AddressPart::Name),
938                    index_hdr: 20,
939                    index_part: -1,
940                })),
941            ),
942            (
943                "${header.from[*].addr}",
944                Value::Variable(VariableType::Header(HeaderVariable {
945                    name: vec![HeaderName::From].into(),
946                    part: HeaderPart::Address(AddressPart::All),
947                    index_hdr: 0,
948                    index_part: -1,
949                })),
950            ),
951            (
952                "${header.from[-5].name[2]}",
953                Value::Variable(VariableType::Header(HeaderVariable {
954                    name: vec![HeaderName::From].into(),
955                    part: HeaderPart::Address(AddressPart::Name),
956                    index_hdr: -5,
957                    index_part: 2,
958                })),
959            ),
960            (
961                "${header.from[*].raw[*]}",
962                Value::Variable(VariableType::Header(HeaderVariable {
963                    name: vec![HeaderName::From].into(),
964                    part: HeaderPart::Raw,
965                    index_hdr: 0,
966                    index_part: 0,
967                })),
968            ),
969        ] {
970            assert_eq!(
971                compiler.tokenize_string(input.as_bytes(), true).unwrap(),
972                expected_result,
973                "Failed for {input}"
974            );
975        }
976
977        for input in ["${unicode:200000}", "${Unicode:DF01}"] {
978            assert!(compiler.tokenize_string(input.as_bytes(), true).is_err());
979        }
980    }
981}
sieve/compiler/lexer/string.rs

sieve/compiler/lexer/
string.rs
