sieve/compiler/lexer/
string.rs

1/*
2 * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
3 *
4 * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
5 */
6
7use std::fmt::Display;
8
9use mail_parser::HeaderName;
10
11use crate::{
12    compiler::{
13        grammar::{
14            expr::{self},
15            instruction::CompilerState,
16            AddressPart,
17        },
18        ContentTypePart, ErrorType, HeaderPart, HeaderVariable, MessagePart, Number,
19        ReceivedHostname, ReceivedPart, Value, VariableType,
20    },
21    runtime::eval::IntoString,
22    Envelope, MAX_MATCH_VARIABLES,
23};
24
25enum State {
26    None,
27    Variable,
28    Encoded {
29        is_unicode: bool,
30        initial_buf_size: usize,
31    },
32}
33
34impl CompilerState<'_> {
35    pub(crate) fn tokenize_string(
36        &mut self,
37        bytes: &[u8],
38        parse_decoded: bool,
39    ) -> Result<Value, ErrorType> {
40        let mut state = State::None;
41        let mut items = Vec::with_capacity(3);
42        let mut last_ch = 0;
43
44        let mut var_start_pos = usize::MAX;
45        let mut var_is_number = true;
46        let mut var_has_namespace = false;
47
48        let mut text_has_digits = true;
49        let mut text_has_dots = false;
50
51        let mut hex_start = usize::MAX;
52        let mut decode_buf = Vec::with_capacity(bytes.len());
53
54        for (pos, &ch) in bytes.iter().enumerate() {
55            let mut is_var_error = false;
56
57            match state {
58                State::None => match ch {
59                    b'{' if last_ch == b'$' => {
60                        decode_buf.pop();
61                        var_start_pos = pos + 1;
62                        var_is_number = true;
63                        var_has_namespace = false;
64                        state = State::Variable;
65                    }
66                    b'.' => {
67                        if text_has_dots {
68                            text_has_digits = false;
69                        } else {
70                            text_has_dots = true;
71                        }
72                        decode_buf.push(ch);
73                    }
74                    b'0'..=b'9' => {
75                        decode_buf.push(ch);
76                    }
77                    _ => {
78                        text_has_digits = false;
79                        decode_buf.push(ch);
80                    }
81                },
82                State::Variable => match ch {
83                    b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'[' | b']' | b'*' | b'-' => {
84                        var_is_number = false;
85                    }
86                    b'.' => {
87                        var_is_number = false;
88                        var_has_namespace = true;
89                    }
90                    b'0'..=b'9' => {}
91                    b'}' => {
92                        if pos > var_start_pos {
93                            // Add any text before the variable
94                            if !decode_buf.is_empty() {
95                                self.add_value(
96                                    &mut items,
97                                    &decode_buf,
98                                    parse_decoded,
99                                    text_has_digits,
100                                    text_has_dots,
101                                )?;
102                                decode_buf.clear();
103                                text_has_digits = true;
104                                text_has_dots = false;
105                            }
106
107                            // Parse variable type
108                            let var_name = std::str::from_utf8(&bytes[var_start_pos..pos]).unwrap();
109                            let var_type = if !var_is_number {
110                                self.parse_variable(var_name, var_has_namespace)
111                            } else {
112                                self.parse_match_variable(var_name)
113                            };
114
115                            match var_type {
116                                Ok(Some(var)) => items.push(Value::Variable(var)),
117                                Ok(None) => {}
118                                Err(
119                                    ErrorType::InvalidNamespace(_) | ErrorType::InvalidEnvelope(_),
120                                ) => {
121                                    is_var_error = true;
122                                }
123                                Err(e) => return Err(e),
124                            }
125
126                            state = State::None;
127                        } else {
128                            is_var_error = true;
129                        }
130                    }
131                    b':' => {
132                        if parse_decoded && !var_has_namespace {
133                            match bytes.get(var_start_pos..pos) {
134                                Some(enc) if enc.eq_ignore_ascii_case(b"hex") => {
135                                    state = State::Encoded {
136                                        is_unicode: false,
137                                        initial_buf_size: decode_buf.len(),
138                                    };
139                                }
140                                Some(enc) if enc.eq_ignore_ascii_case(b"unicode") => {
141                                    state = State::Encoded {
142                                        is_unicode: true,
143                                        initial_buf_size: decode_buf.len(),
144                                    };
145                                }
146                                _ => {
147                                    is_var_error = true;
148                                }
149                            }
150                        } else if var_has_namespace {
151                            var_is_number = false;
152                        } else {
153                            is_var_error = true;
154                        }
155                    }
156                    _ => {
157                        is_var_error = true;
158                    }
159                },
160
161                State::Encoded {
162                    is_unicode,
163                    initial_buf_size,
164                } => match ch {
165                    b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => {
166                        if hex_start == usize::MAX {
167                            hex_start = pos;
168                        }
169                    }
170                    b' ' | b'\t' | b'\r' | b'\n' | b'}' => {
171                        if hex_start != usize::MAX {
172                            let code = std::str::from_utf8(&bytes[hex_start..pos]).unwrap();
173                            hex_start = usize::MAX;
174
175                            if !is_unicode {
176                                if let Ok(ch) = u8::from_str_radix(code, 16) {
177                                    decode_buf.push(ch);
178                                } else {
179                                    is_var_error = true;
180                                }
181                            } else if let Ok(ch) = u32::from_str_radix(code, 16) {
182                                let mut buf = [0; 4];
183                                decode_buf.extend_from_slice(
184                                    char::from_u32(ch)
185                                        .ok_or(ErrorType::InvalidUnicodeSequence(ch))?
186                                        .encode_utf8(&mut buf)
187                                        .as_bytes(),
188                                );
189                            } else {
190                                is_var_error = true;
191                            }
192                        }
193                        if ch == b'}' {
194                            if decode_buf.len() != initial_buf_size {
195                                state = State::None;
196                            } else {
197                                is_var_error = true;
198                            }
199                        }
200                    }
201                    _ => {
202                        is_var_error = true;
203                    }
204                },
205            }
206
207            if is_var_error {
208                if let State::Encoded {
209                    initial_buf_size, ..
210                } = state
211                {
212                    if initial_buf_size != decode_buf.len() {
213                        decode_buf.truncate(initial_buf_size);
214                    }
215                }
216                decode_buf.extend_from_slice(&bytes[var_start_pos - 2..pos + 1]);
217                hex_start = usize::MAX;
218                state = State::None;
219            }
220
221            last_ch = ch;
222        }
223
224        match state {
225            State::Variable => {
226                decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]);
227            }
228            State::Encoded {
229                initial_buf_size, ..
230            } => {
231                if initial_buf_size != decode_buf.len() {
232                    decode_buf.truncate(initial_buf_size);
233                }
234                decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]);
235            }
236            State::None => (),
237        }
238
239        if !decode_buf.is_empty() {
240            self.add_value(
241                &mut items,
242                &decode_buf,
243                parse_decoded,
244                text_has_digits,
245                text_has_dots,
246            )?;
247        }
248
249        Ok(match items.len() {
250            1 => items.pop().unwrap(),
251            0 => Value::Text(String::new().into()),
252            _ => Value::List(items),
253        })
254    }
255
256    fn parse_match_variable(&mut self, var_name: &str) -> Result<Option<VariableType>, ErrorType> {
257        let num = var_name
258            .parse()
259            .map_err(|_| ErrorType::InvalidNumber(var_name.to_string()))?;
260        if num < MAX_MATCH_VARIABLES as usize {
261            if self.register_match_var(num) {
262                let total_vars = num + 1;
263                if total_vars > self.vars_match_max {
264                    self.vars_match_max = total_vars;
265                }
266                Ok(Some(VariableType::Match(num)))
267            } else {
268                Ok(None)
269            }
270        } else {
271            Err(ErrorType::InvalidMatchVariable(num))
272        }
273    }
274
275    pub fn parse_variable(
276        &self,
277        var_name: &str,
278        maybe_namespace: bool,
279    ) -> Result<Option<VariableType>, ErrorType> {
280        if !maybe_namespace {
281            if self.is_var_global(var_name) {
282                Ok(Some(VariableType::Global(var_name.to_string())))
283            } else if let Some(var_id) = self.get_local_var(var_name) {
284                Ok(Some(VariableType::Local(var_id)))
285            } else {
286                Ok(None)
287            }
288        } else {
289            let var = match var_name.to_lowercase().split_once('.') {
290                Some(("global" | "t", var_name)) if !var_name.is_empty() => {
291                    VariableType::Global(var_name.to_string())
292                }
293                Some(("env", var_name)) if !var_name.is_empty() => {
294                    VariableType::Environment(var_name.to_string())
295                }
296                Some(("envelope", var_name)) if !var_name.is_empty() => {
297                    let envelope = match var_name {
298                        "from" => Envelope::From,
299                        "to" => Envelope::To,
300                        "by_time_absolute" => Envelope::ByTimeAbsolute,
301                        "by_time_relative" => Envelope::ByTimeRelative,
302                        "by_mode" => Envelope::ByMode,
303                        "by_trace" => Envelope::ByTrace,
304                        "notify" => Envelope::Notify,
305                        "orcpt" => Envelope::Orcpt,
306                        "ret" => Envelope::Ret,
307                        "envid" => Envelope::Envid,
308                        _ => {
309                            return Err(ErrorType::InvalidEnvelope(var_name.to_string()));
310                        }
311                    };
312                    VariableType::Envelope(envelope)
313                }
314                Some(("header", var_name)) if !var_name.is_empty() => {
315                    self.parse_header_variable(var_name)?
316                }
317                Some(("body", var_name)) if !var_name.is_empty() => match var_name {
318                    "text" => VariableType::Part(MessagePart::TextBody(false)),
319                    "html" => VariableType::Part(MessagePart::HtmlBody(false)),
320                    "to_text" => VariableType::Part(MessagePart::TextBody(true)),
321                    "to_html" => VariableType::Part(MessagePart::HtmlBody(true)),
322                    _ => return Err(ErrorType::InvalidNamespace(var_name.to_string())),
323                },
324                Some(("part", var_name)) if !var_name.is_empty() => match var_name {
325                    "text" => VariableType::Part(MessagePart::Contents),
326                    "raw" => VariableType::Part(MessagePart::Raw),
327                    _ => return Err(ErrorType::InvalidNamespace(var_name.to_string())),
328                },
329                None => {
330                    if self.is_var_global(var_name) {
331                        VariableType::Global(var_name.to_string())
332                    } else if let Some(var_id) = self.get_local_var(var_name) {
333                        VariableType::Local(var_id)
334                    } else {
335                        return Ok(None);
336                    }
337                }
338                _ => return Err(ErrorType::InvalidNamespace(var_name.to_string())),
339            };
340
341            Ok(Some(var))
342        }
343    }
344
345    fn parse_header_variable(&self, var_name: &str) -> Result<VariableType, ErrorType> {
346        #[derive(Debug)]
347        enum State {
348            Name,
349            Index,
350            Part,
351            PartIndex,
352        }
353        let mut name = vec![];
354        let mut has_name = false;
355        let mut has_wildcard = false;
356        let mut hdr_name = String::new();
357        let mut hdr_index = String::new();
358        let mut part = String::new();
359        let mut part_index = String::new();
360        let mut state = State::Name;
361
362        for ch in var_name.chars() {
363            match state {
364                State::Name => match ch {
365                    '[' => {
366                        state = if hdr_index.is_empty() {
367                            State::Index
368                        } else if part.is_empty() {
369                            State::PartIndex
370                        } else {
371                            return Err(ErrorType::InvalidExpression(var_name.to_string()));
372                        };
373                        has_name = true;
374                    }
375                    '.' => {
376                        state = State::Part;
377                        has_name = true;
378                    }
379                    ' ' | '\t' | '\r' | '\n' => {}
380                    '*' if !has_wildcard && hdr_name.is_empty() && name.is_empty() => {
381                        has_wildcard = true;
382                    }
383                    ':' if !hdr_name.is_empty() && !has_wildcard => {
384                        name.push(
385                            HeaderName::parse(std::mem::take(&mut hdr_name)).ok_or_else(|| {
386                                ErrorType::InvalidExpression(var_name.to_string())
387                            })?,
388                        );
389                    }
390                    _ if !has_name && !has_wildcard => {
391                        hdr_name.push(ch);
392                    }
393                    _ => {
394                        return Err(ErrorType::InvalidExpression(var_name.to_string()));
395                    }
396                },
397                State::Index => match ch {
398                    ']' => {
399                        state = State::Name;
400                    }
401                    ' ' | '\t' | '\r' | '\n' => {}
402                    _ => {
403                        hdr_index.push(ch);
404                    }
405                },
406                State::Part => match ch {
407                    '[' => {
408                        state = State::PartIndex;
409                    }
410                    ' ' | '\t' | '\r' | '\n' => {}
411                    _ => {
412                        part.push(ch);
413                    }
414                },
415                State::PartIndex => match ch {
416                    ']' => {
417                        state = State::Name;
418                    }
419                    ' ' | '\t' | '\r' | '\n' => {}
420                    _ => {
421                        part_index.push(ch);
422                    }
423                },
424            }
425        }
426
427        if !hdr_name.is_empty() {
428            name.push(
429                HeaderName::parse(hdr_name)
430                    .ok_or_else(|| ErrorType::InvalidExpression(var_name.to_string()))?,
431            );
432        }
433
434        if !name.is_empty() || has_wildcard {
435            Ok(VariableType::Header(HeaderVariable {
436                name,
437                part: HeaderPart::try_from(part.as_str())
438                    .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
439                index_hdr: match hdr_index.as_str() {
440                    "" => {
441                        if !has_wildcard {
442                            -1
443                        } else {
444                            0
445                        }
446                    }
447                    "*" => 0,
448                    _ => hdr_index
449                        .parse()
450                        .map(|v| if v == 0 { 1 } else { v })
451                        .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
452                },
453                index_part: match part_index.as_str() {
454                    "" => {
455                        if !has_wildcard {
456                            -1
457                        } else {
458                            0
459                        }
460                    }
461                    "*" => 0,
462                    _ => part_index
463                        .parse()
464                        .map(|v| if v == 0 { 1 } else { v })
465                        .map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
466                },
467            }))
468        } else {
469            Err(ErrorType::InvalidExpression(var_name.to_string()))
470        }
471    }
472
473    pub fn parse_expr_fnc_or_var(
474        &self,
475        var_name: &str,
476        maybe_namespace: bool,
477    ) -> Result<expr::Token, String> {
478        match self.parse_variable(var_name, maybe_namespace) {
479            Ok(Some(var)) => Ok(expr::Token::Variable(var)),
480            _ => {
481                if let Some((id, num_args)) = self.compiler.functions.get(var_name) {
482                    Ok(expr::Token::Function {
483                        name: var_name.to_string(),
484                        id: *id,
485                        num_args: *num_args,
486                    })
487                } else {
488                    Err(format!("Invalid variable or function name {var_name:?}"))
489                }
490            }
491        }
492    }
493
494    #[inline(always)]
495    fn add_value(
496        &mut self,
497        items: &mut Vec<Value>,
498        buf: &[u8],
499        parse_decoded: bool,
500        has_digits: bool,
501        has_dots: bool,
502    ) -> Result<(), ErrorType> {
503        if !parse_decoded {
504            items.push(if has_digits {
505                if has_dots {
506                    match std::str::from_utf8(buf)
507                        .ok()
508                        .and_then(|v| (v, v.parse::<f64>().ok()?).into())
509                    {
510                        Some((v, n)) if n.to_string() == v => Value::Number(Number::Float(n)),
511                        _ => Value::Text(buf.to_vec().into_string().into()),
512                    }
513                } else {
514                    match std::str::from_utf8(buf)
515                        .ok()
516                        .and_then(|v| (v, v.parse::<i64>().ok()?).into())
517                    {
518                        Some((v, n)) if n.to_string() == v => Value::Number(Number::Integer(n)),
519                        _ => Value::Text(buf.to_vec().into_string().into()),
520                    }
521                }
522            } else {
523                Value::Text(buf.to_vec().into_string().into())
524            });
525        } else {
526            match self.tokenize_string(buf, false)? {
527                Value::List(new_items) => items.extend(new_items),
528                item => items.push(item),
529            }
530        }
531
532        Ok(())
533    }
534}
535
536impl TryFrom<&str> for HeaderPart {
537    type Error = ();
538
539    fn try_from(value: &str) -> Result<Self, Self::Error> {
540        let (value, subvalue) = value.split_once('.').unwrap_or((value, ""));
541        Ok(match value {
542            "" | "text" => HeaderPart::Text,
543            // Addresses
544            "name" => HeaderPart::Address(AddressPart::Name),
545            "addr" => {
546                if !subvalue.is_empty() {
547                    HeaderPart::Address(AddressPart::try_from(subvalue)?)
548                } else {
549                    HeaderPart::Address(AddressPart::All)
550                }
551            }
552
553            // Content-type
554            "type" => HeaderPart::ContentType(ContentTypePart::Type),
555            "subtype" => HeaderPart::ContentType(ContentTypePart::Subtype),
556            "attr" if !subvalue.is_empty() => {
557                HeaderPart::ContentType(ContentTypePart::Attribute(subvalue.to_string()))
558            }
559
560            // Received
561            "rcvd" => {
562                if !subvalue.is_empty() {
563                    HeaderPart::Received(ReceivedPart::try_from(subvalue)?)
564                } else {
565                    HeaderPart::Text
566                }
567            }
568
569            // Id
570            "id" => HeaderPart::Id,
571
572            // Raw
573            "raw" => HeaderPart::Raw,
574            "raw_name" => HeaderPart::RawName,
575
576            // Date
577            "date" => HeaderPart::Date,
578
579            // Exists
580            "exists" => HeaderPart::Exists,
581
582            _ => {
583                return Err(());
584            }
585        })
586    }
587}
588
589impl TryFrom<&str> for ReceivedPart {
590    type Error = ();
591
592    fn try_from(value: &str) -> Result<Self, Self::Error> {
593        Ok(match value {
594            // Received
595            "from" => ReceivedPart::From(ReceivedHostname::Any),
596            "from.name" => ReceivedPart::From(ReceivedHostname::Name),
597            "from.ip" => ReceivedPart::From(ReceivedHostname::Ip),
598            "ip" => ReceivedPart::FromIp,
599            "iprev" => ReceivedPart::FromIpRev,
600            "by" => ReceivedPart::By(ReceivedHostname::Any),
601            "by.name" => ReceivedPart::By(ReceivedHostname::Name),
602            "by.ip" => ReceivedPart::By(ReceivedHostname::Ip),
603            "for" => ReceivedPart::For,
604            "with" => ReceivedPart::With,
605            "tls" => ReceivedPart::TlsVersion,
606            "cipher" => ReceivedPart::TlsCipher,
607            "id" => ReceivedPart::Id,
608            "ident" => ReceivedPart::Ident,
609            "date" => ReceivedPart::Date,
610            "date.raw" => ReceivedPart::DateRaw,
611            _ => return Err(()),
612        })
613    }
614}
615
616impl TryFrom<&str> for AddressPart {
617    type Error = ();
618
619    fn try_from(value: &str) -> Result<Self, Self::Error> {
620        Ok(match value {
621            "name" => AddressPart::Name,
622            "addr" | "all" => AddressPart::All,
623            "addr.domain" => AddressPart::Domain,
624            "addr.local" => AddressPart::LocalPart,
625            "addr.user" => AddressPart::User,
626            "addr.detail" => AddressPart::Detail,
627            _ => return Err(()),
628        })
629    }
630}
631
632impl Display for Value {
633    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
634        match self {
635            Value::Text(t) => f.write_str(t),
636            Value::List(l) => {
637                for i in l {
638                    i.fmt(f)?;
639                }
640                Ok(())
641            }
642            Value::Number(n) => n.fmt(f),
643            Value::Variable(v) => v.fmt(f),
644            Value::Regex(r) => f.write_str(&r.expr),
645        }
646    }
647}
648
649impl Display for VariableType {
650    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
651        match self {
652            VariableType::Local(v) => write!(f, "${{{v}}}"),
653            VariableType::Match(v) => write!(f, "${{{v}}}"),
654            VariableType::Global(v) => write!(f, "${{global.{v}}}"),
655            VariableType::Environment(v) => write!(f, "${{env.{v}}}"),
656
657            VariableType::Envelope(env) => f.write_str(match env {
658                Envelope::From => "${{envelope.from}}",
659                Envelope::To => "${{envelope.to}}",
660                Envelope::ByTimeAbsolute => "${{envelope.by_time_absolute}}",
661                Envelope::ByTimeRelative => "${{envelope.by_time_relative}}",
662                Envelope::ByMode => "${{envelope.by_mode}}",
663                Envelope::ByTrace => "${{envelope.by_trace}}",
664                Envelope::Notify => "${{envelope.notify}}",
665                Envelope::Orcpt => "${{envelope.orcpt}}",
666                Envelope::Ret => "${{envelope.ret}}",
667                Envelope::Envid => "${{envelope.envit}}",
668            }),
669
670            VariableType::Header(hdr) => {
671                write!(
672                    f,
673                    "${{header.{}",
674                    hdr.name.first().map(|h| h.as_str()).unwrap_or_default()
675                )?;
676                if hdr.index_hdr != 0 {
677                    write!(f, "[{}]", hdr.index_hdr)?;
678                } else {
679                    f.write_str("[*]")?;
680                }
681                /*if hdr.part != HeaderPart::Text {
682                    f.write_str(".")?;
683                    f.write_str(match &hdr.part {
684                        HeaderPart::Name => "name",
685                        HeaderPart::Address => "address",
686                        HeaderPart::Type => "type",
687                        HeaderPart::Subtype => "subtype",
688                        HeaderPart::Raw => "raw",
689                        HeaderPart::Date => "date",
690                        HeaderPart::Attribute(attr) => attr.as_str(),
691                        HeaderPart::Text => unreachable!(),
692                    })?;
693                }*/
694                if hdr.index_part != 0 {
695                    write!(f, "[{}]", hdr.index_part)?;
696                } else {
697                    f.write_str("[*]")?;
698                }
699                f.write_str("}")
700            }
701            VariableType::Part(part) => {
702                write!(
703                    f,
704                    "${{{}",
705                    match part {
706                        MessagePart::TextBody(true) => "body.to_text",
707                        MessagePart::TextBody(false) => "body.text",
708                        MessagePart::HtmlBody(true) => "body.to_html",
709                        MessagePart::HtmlBody(false) => "body.html",
710                        MessagePart::Contents => "part.text",
711                        MessagePart::Raw => "part.raw",
712                    }
713                )?;
714                f.write_str("}")
715            }
716        }
717    }
718}
719
720#[cfg(test)]
721mod tests {
722
723    use mail_parser::HeaderName;
724
725    use super::Value;
726    use crate::compiler::grammar::instruction::{Block, CompilerState, Instruction, MAX_PARAMS};
727    use crate::compiler::grammar::test::Test;
728    use crate::compiler::grammar::tests::test_string::TestString;
729    use crate::compiler::grammar::{Comparator, MatchType};
730    use crate::compiler::lexer::tokenizer::Tokenizer;
731    use crate::compiler::lexer::word::Word;
732    use crate::compiler::{AddressPart, HeaderPart, HeaderVariable, VariableType};
733    use crate::{AHashSet, Compiler};
734
735    #[test]
736    fn tokenize_string() {
737        let c = Compiler::new();
738        let mut block = Block::new(Word::Not);
739        block.match_test_pos.push(0);
740        let mut compiler = CompilerState {
741            compiler: &c,
742            instructions: vec![Instruction::Test(Test::String(TestString {
743                match_type: MatchType::Regex(u64::MAX),
744                comparator: Comparator::AsciiCaseMap,
745                source: vec![Value::Variable(VariableType::Local(0))],
746                key_list: vec![Value::Variable(VariableType::Local(0))],
747                is_not: false,
748            }))],
749            block_stack: Vec::new(),
750            block,
751            last_block_type: Word::Not,
752            vars_global: AHashSet::new(),
753            vars_num: 0,
754            vars_num_max: 0,
755            vars_local: 0,
756            tokens: Tokenizer::new(&c, b""),
757            vars_match_max: usize::MAX,
758            param_check: [false; MAX_PARAMS],
759            includes_num: 0,
760        };
761
762        for (input, expected_result) in [
763            ("$${hex:24 24}", Value::Text("$$$".to_string().into())),
764            ("$${hex:40}", Value::Text("$@".to_string().into())),
765            ("${hex: 40 }", Value::Text("@".to_string().into())),
766            ("${HEX: 40}", Value::Text("@".to_string().into())),
767            ("${hex:40", Value::Text("${hex:40".to_string().into())),
768            ("${hex:400}", Value::Text("${hex:400}".to_string().into())),
769            (
770                "${hex:4${hex:30}}",
771                Value::Text("${hex:40}".to_string().into()),
772            ),
773            ("${unicode:40}", Value::Text("@".to_string().into())),
774            (
775                "${ unicode:40}",
776                Value::Text("${ unicode:40}".to_string().into()),
777            ),
778            ("${UNICODE:40}", Value::Text("@".to_string().into())),
779            ("${UnICoDE:0000040}", Value::Text("@".to_string().into())),
780            ("${Unicode:40}", Value::Text("@".to_string().into())),
781            (
782                "${Unicode:40 40 ",
783                Value::Text("${Unicode:40 40 ".to_string().into()),
784            ),
785            (
786                "${Unicode:Cool}",
787                Value::Text("${Unicode:Cool}".to_string().into()),
788            ),
789            ("", Value::Text("".to_string().into())),
790            (
791                "${global.full}",
792                Value::Variable(VariableType::Global("full".to_string())),
793            ),
794            (
795                "${BAD${global.Company}",
796                Value::List(vec![
797                    Value::Text("${BAD".to_string().into()),
798                    Value::Variable(VariableType::Global("company".to_string())),
799                ]),
800            ),
801            (
802                "${President, ${global.Company} Inc.}",
803                Value::List(vec![
804                    Value::Text("${President, ".to_string().into()),
805                    Value::Variable(VariableType::Global("company".to_string())),
806                    Value::Text(" Inc.}".to_string().into()),
807                ]),
808            ),
809            (
810                "dear${hex:20 24 7b}global.Name}",
811                Value::List(vec![
812                    Value::Text("dear ".to_string().into()),
813                    Value::Variable(VariableType::Global("name".to_string())),
814                ]),
815            ),
816            (
817                "INBOX.lists.${2}",
818                Value::List(vec![
819                    Value::Text("INBOX.lists.".to_string().into()),
820                    Value::Variable(VariableType::Match(2)),
821                ]),
822            ),
823            (
824                "Ein unerh${unicode:00F6}rt gro${unicode:00DF}er Test",
825                Value::Text("Ein unerhört großer Test".to_string().into()),
826            ),
827            ("&%${}!", Value::Text("&%${}!".to_string().into())),
828            ("${doh!}", Value::Text("${doh!}".to_string().into())),
829            (
830                "${hex: 20 }${global.hi}${hex: 20 }",
831                Value::List(vec![
832                    Value::Text(" ".to_string().into()),
833                    Value::Variable(VariableType::Global("hi".to_string())),
834                    Value::Text(" ".to_string().into()),
835                ]),
836            ),
837            (
838                "${hex:20 24 7b z}${global.hi}${unicode:}${unicode: }${hex:20}",
839                Value::List(vec![
840                    Value::Text("${hex:20 24 7b z}".to_string().into()),
841                    Value::Variable(VariableType::Global("hi".to_string())),
842                    Value::Text("${unicode:}${unicode: } ".to_string().into()),
843                ]),
844            ),
845            (
846                "${header.from}",
847                Value::Variable(VariableType::Header(HeaderVariable {
848                    name: vec![HeaderName::From],
849                    part: HeaderPart::Text,
850                    index_hdr: -1,
851                    index_part: -1,
852                })),
853            ),
854            (
855                "${header.from.addr}",
856                Value::Variable(VariableType::Header(HeaderVariable {
857                    name: vec![HeaderName::From],
858                    part: HeaderPart::Address(AddressPart::All),
859                    index_hdr: -1,
860                    index_part: -1,
861                })),
862            ),
863            (
864                "${header.from[1]}",
865                Value::Variable(VariableType::Header(HeaderVariable {
866                    name: vec![HeaderName::From],
867                    part: HeaderPart::Text,
868                    index_hdr: 1,
869                    index_part: -1,
870                })),
871            ),
872            (
873                "${header.from[*]}",
874                Value::Variable(VariableType::Header(HeaderVariable {
875                    name: vec![HeaderName::From],
876                    part: HeaderPart::Text,
877                    index_hdr: 0,
878                    index_part: -1,
879                })),
880            ),
881            (
882                "${header.from[20].name}",
883                Value::Variable(VariableType::Header(HeaderVariable {
884                    name: vec![HeaderName::From],
885                    part: HeaderPart::Address(AddressPart::Name),
886                    index_hdr: 20,
887                    index_part: -1,
888                })),
889            ),
890            (
891                "${header.from[*].addr}",
892                Value::Variable(VariableType::Header(HeaderVariable {
893                    name: vec![HeaderName::From],
894                    part: HeaderPart::Address(AddressPart::All),
895                    index_hdr: 0,
896                    index_part: -1,
897                })),
898            ),
899            (
900                "${header.from[-5].name[2]}",
901                Value::Variable(VariableType::Header(HeaderVariable {
902                    name: vec![HeaderName::From],
903                    part: HeaderPart::Address(AddressPart::Name),
904                    index_hdr: -5,
905                    index_part: 2,
906                })),
907            ),
908            (
909                "${header.from[*].raw[*]}",
910                Value::Variable(VariableType::Header(HeaderVariable {
911                    name: vec![HeaderName::From],
912                    part: HeaderPart::Raw,
913                    index_hdr: 0,
914                    index_part: 0,
915                })),
916            ),
917        ] {
918            assert_eq!(
919                compiler.tokenize_string(input.as_bytes(), true).unwrap(),
920                expected_result,
921                "Failed for {input}"
922            );
923        }
924
925        for input in ["${unicode:200000}", "${Unicode:DF01}"] {
926            assert!(compiler.tokenize_string(input.as_bytes(), true).is_err());
927        }
928    }
929}
sieve/compiler/lexer/string.rs

sieve/compiler/lexer/
string.rs
