pochoir_lang/
parser.rs

1// `Spanned::with_span` does not allow using `RangeInclusive`
2#![allow(clippy::range_plus_one)]
3
4use crate::{error::AutoErrorOffset, Error, Result};
5
6use indexmap::IndexMap;
7use pochoir_common::{Spanned, StreamParser};
8use std::{borrow::Cow, path::Path};
9
10#[derive(Debug, PartialEq, Clone, Copy)]
11pub enum UnaryOp {
12    BoolNot,
13    Negate,
14    Range,
15    RangeInclusive,
16}
17
18#[derive(Debug, PartialEq, Clone, Copy)]
19pub enum BinaryOp {
20    Add,
21    Subtract,
22    Multiply,
23    Divide,
24
25    Equal,
26    NotEqual,
27
28    Greater,
29    GreaterOrEqual,
30    Less,
31    LessOrEqual,
32
33    Pipe,
34
35    BoolAnd,
36    BoolOr,
37
38    Semicolon,
39    Definition,
40    Range,
41    RangeInclusive,
42}
43
44#[derive(Debug, PartialEq, Clone, Copy)]
45pub enum TernaryOp {
46    Conditional,
47}
48
49#[derive(Debug, PartialEq, Clone)]
50pub enum Literal<'a> {
51    String(String),
52    Number(f64),
53    Array(Vec<Vec<Spanned<Token<'a>>>>),
54    Object(IndexMap<String, Vec<Spanned<Token<'a>>>>),
55    Bool(bool),
56    Null,
57}
58
59#[derive(Debug, PartialEq, Clone)]
60pub enum Token<'a> {
61    UnaryOperator(UnaryOp, Vec<Spanned<Token<'a>>>),
62    BinaryOperator(BinaryOp, Vec<Spanned<Token<'a>>>, Vec<Spanned<Token<'a>>>),
63    TernaryOperator(
64        TernaryOp,
65        Vec<Spanned<Token<'a>>>,
66        Vec<Spanned<Token<'a>>>,
67        Vec<Spanned<Token<'a>>>,
68    ),
69    Literal(Literal<'a>),
70    FunctionCall(Box<Spanned<Token<'a>>>, Vec<Vec<Spanned<Token<'a>>>>),
71    Variable(Cow<'a, str>),
72    Index(Vec<Spanned<Token<'a>>>, Box<Spanned<Token<'a>>>),
73}
74
75/// Parse a string into a list of tokens.
76///
77/// `file_path` is used in errors and `file_offset` corresponds to the offset between the start of
78/// the "real" contents and the chunk of text that was given to this function.
79///
80/// ### Errors
81///
82/// Returns an [`Error`] representing an error which happened when parsing the source.
83///
84/// [`Error`]: crate::Error
85pub fn parse<P: AsRef<Path>>(
86    file_path: P,
87    value: &str,
88    file_offset: usize,
89) -> Result<Vec<Spanned<Token<'_>>>> {
90    let file_path = file_path.as_ref();
91    let mut parser = StreamParser::new(file_path, value);
92    let mut tokens = vec![];
93
94    while !parser.is_eoi() {
95        parse_recursive(&mut parser, &mut tokens, 0, file_offset)?;
96    }
97
98    Ok(tokens)
99}
100
101// A basic Pratt parser, based on the awesome blog post
102// https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html
103fn parse_recursive<'a>(
104    parser: &mut StreamParser<'a>,
105    tokens: &mut Vec<Spanned<Token<'a>>>,
106    min_precedence: usize,
107    file_offset: usize,
108) -> Result<()> {
109    parser.trim();
110
111    let start = parser.index();
112    let mut left_tokens = vec![];
113
114    if parser.take_exact("(").is_ok() {
115        parse_recursive(parser, &mut left_tokens, 0, file_offset)?;
116        parser.take_exact(")").map_err(|_e| {
117            Spanned::new(Error::MismatchedClosingDelimiter(")".to_string()))
118                .with_span(start + file_offset..start + file_offset + 1)
119                .with_file_path(parser.file_path())
120        })?;
121    } else if let Some((operator, length)) = parse_unary_operator(parser) {
122        // Left part of operator with an operator before
123        parser.take(length).auto_error_offset(file_offset)?;
124
125        let mut tokens = vec![];
126        parse_recursive(parser, &mut tokens, 1242, file_offset)?;
127
128        let end = parser.index();
129        left_tokens.push(
130            Spanned::new(Token::UnaryOperator(operator, tokens))
131                .with_span(start + file_offset..end + file_offset)
132                .with_file_path(parser.file_path()),
133        );
134    } else {
135        // Left part of operator
136        left_tokens.push(parse_token(parser, file_offset)?);
137    }
138
139    loop {
140        let before_trim_index = parser.index();
141        parser.trim();
142
143        // Binary operator
144        if let Some((operator, length, (left_precedence, right_precedence))) =
145            parse_binary_operator(parser)
146        {
147            if left_precedence < min_precedence {
148                parser.set_index(before_trim_index);
149                break;
150            }
151
152            parser.take(length).auto_error_offset(file_offset)?;
153            parser.trim();
154
155            // Right part of operator
156            let mut right_tokens = vec![];
157            if operator == BinaryOp::Range || operator == BinaryOp::RangeInclusive {
158                // Directly return if it is the end of a special group, used as a fast case
159                if !parser
160                    .next()
161                    .map_or(true, |ch| [')', ']', '}', ';'].contains(&ch))
162                {
163                    parse_recursive(parser, &mut right_tokens, right_precedence, file_offset)?;
164                }
165            } else {
166                parse_recursive(parser, &mut right_tokens, right_precedence, file_offset)?;
167            }
168            let end = parser.index();
169
170            left_tokens =
171                vec![
172                    Spanned::new(Token::BinaryOperator(operator, left_tokens, right_tokens))
173                        .with_span(start + file_offset..end + file_offset)
174                        .with_file_path(parser.file_path()),
175                ];
176            continue;
177        }
178
179        // Ternary operator
180        if let Some((operator, length, (left_precedence, right_precedence))) =
181            parse_ternary_operator(parser)
182        {
183            if left_precedence < min_precedence {
184                parser.set_index(before_trim_index);
185                break;
186            }
187
188            parser.take(length).auto_error_offset(file_offset)?;
189            parser.trim();
190
191            // Middle part of operator
192            let mut middle_tokens = vec![];
193            parse_recursive(parser, &mut middle_tokens, 0, file_offset)?;
194
195            parser.trim();
196            parser.take_exact(":").auto_error_offset(file_offset)?;
197            parser.trim();
198
199            // Right part of operator
200            let mut right_tokens = vec![];
201            parse_recursive(parser, &mut right_tokens, right_precedence, file_offset)?;
202
203            let end = parser.index();
204
205            left_tokens = vec![Spanned::new(Token::TernaryOperator(
206                operator,
207                left_tokens,
208                middle_tokens,
209                right_tokens,
210            ))
211            .with_span(start + file_offset..end + file_offset)
212            .with_file_path(parser.file_path())];
213
214            continue;
215        }
216
217        parser.set_index(before_trim_index);
218        break;
219    }
220
221    tokens.extend(left_tokens);
222    Ok(())
223}
224
225#[allow(clippy::too_many_lines)]
226fn parse_token<'a>(
227    parser: &mut StreamParser<'a>,
228    file_offset: usize,
229) -> Result<Spanned<Token<'a>>> {
230    let token_start = parser.index();
231    let mut token = None;
232    parser.trim();
233
234    // Literals
235    match parser.next().auto_error_offset(file_offset)? {
236        '"' | '\'' => token = Some(Token::Literal(string(parser, file_offset)?)),
237        ch if ch.is_ascii_digit() => token = Some(Token::Literal(number(parser)?)),
238        '[' => token = Some(Token::Literal(array(parser, file_offset)?)),
239        '{' => token = Some(Token::Literal(object(parser, file_offset)?)),
240        _ => (),
241    }
242
243    if parser.take_exact("true").is_ok() {
244        token = Some(Token::Literal(Literal::Bool(true)));
245    } else if parser.take_exact("false").is_ok() {
246        token = Some(Token::Literal(Literal::Bool(false)));
247    } else if parser.take_exact("null").is_ok() {
248        token = Some(Token::Literal(Literal::Null));
249    }
250
251    // Identifier (mostly a variable or the name of a function)
252    if parser
253        .next()
254        .is_ok_and(|ch| ch.is_alphabetic() || ch == '_')
255    {
256        let identifier_name = parser.take_while(|(_, ch)| ch.is_alphanumeric() || ch == '_');
257        token = Some(Token::Variable(Cow::Borrowed(identifier_name)));
258    }
259
260    if let Some(mut token) = token {
261        loop {
262            // Index
263            loop {
264                // Avoid parsing ranges as index
265                if parser.peek_exact(".") && !parser.peek_early_exact(".", 1) {
266                    parser.take_exact(".").unwrap();
267
268                    let index_start = parser.index();
269                    let index_name = parser.take_while(|(_, ch)| ch.is_alphanumeric() || ch == '_');
270                    let index_end = parser.index();
271
272                    token = Token::Index(
273                        vec![
274                            Spanned::new(Token::Literal(Literal::String(index_name.to_string())))
275                                .with_span(index_start + file_offset..index_end + file_offset)
276                                .with_file_path(parser.file_path()),
277                        ],
278                        Box::new(
279                            Spanned::new(token)
280                                .with_span(token_start + file_offset..index_start + file_offset - 1)
281                                .with_file_path(parser.file_path()),
282                        ),
283                    );
284                } else if parser.take_exact("[").is_ok() {
285                    let index_start = parser.index();
286                    let mut index_tokens = vec![];
287
288                    parser.trim();
289                    parse_recursive(parser, &mut index_tokens, 0, file_offset)?;
290                    parser.trim();
291
292                    parser.take_exact("]").map_err(|_e| {
293                        Spanned::new(Error::MismatchedClosingDelimiter("]".to_string()))
294                            .with_span(index_start + file_offset - 1..index_start + file_offset)
295                            .with_file_path(parser.file_path())
296                    })?;
297
298                    token = Token::Index(
299                        index_tokens,
300                        Box::new(
301                            Spanned::new(token)
302                                .with_span(token_start + file_offset..index_start + file_offset - 1)
303                                .with_file_path(parser.file_path()),
304                        ),
305                    );
306                } else {
307                    break;
308                }
309            }
310
311            // Function
312            if parser.take_exact("(").is_ok() {
313                let function_name_end = parser.index() - 1;
314                parser.trim();
315                let mut args = vec![];
316
317                loop {
318                    parser.trim();
319
320                    if parser.peek_exact(")") {
321                        break;
322                    }
323
324                    let mut tokens_for_arg = vec![];
325                    parse_recursive(parser, &mut tokens_for_arg, 0, file_offset)?;
326                    args.push(tokens_for_arg);
327
328                    parser.trim();
329                    if parser.take_exact(",").is_err() {
330                        break;
331                    }
332                }
333
334                token = Token::FunctionCall(
335                    Box::new(
336                        Spanned::new(token)
337                            .with_span(token_start + file_offset..function_name_end + file_offset)
338                            .with_file_path(parser.file_path()),
339                    ),
340                    args,
341                );
342
343                parser.trim();
344                parser.take_exact(")").map_err(|_e| {
345                    Spanned::new(Error::MismatchedClosingDelimiter(")".to_string()))
346                        .with_span(
347                            function_name_end + file_offset..function_name_end + file_offset + 1,
348                        )
349                        .with_file_path(parser.file_path())
350                })?;
351            } else {
352                break;
353            }
354        }
355
356        let token_end = parser.index();
357        Ok(Spanned::new(token)
358            .with_span(token_start + file_offset..token_end + file_offset)
359            .with_file_path(parser.file_path()))
360    } else {
361        Err(Spanned::new(Error::ExpectedExpression(
362            parser.next().auto_error_offset(file_offset)?.to_string(),
363        ))
364        .with_span(token_start + file_offset..token_start + file_offset + 1)
365        .with_file_path(parser.file_path()))
366    }
367}
368
369fn parse_unary_operator(parser: &mut StreamParser) -> Option<(UnaryOp, usize)> {
370    parser.trim();
371
372    // Unary operators have a precedence power of 1242
373    match parser.next() {
374        Ok('-') => Some((UnaryOp::Negate, 1)),
375        Ok('!') => Some((UnaryOp::BoolNot, 1)),
376        Ok('.') if parser.peek_early(1, 1) == Ok(".") && parser.peek_early(1, 2) == Ok("=") => {
377            Some((UnaryOp::RangeInclusive, 3))
378        }
379        Ok('.') if parser.peek_early(1, 1) == Ok(".") => Some((UnaryOp::Range, 2)),
380        _ => None,
381    }
382}
383
384fn parse_binary_operator(parser: &mut StreamParser) -> Option<(BinaryOp, usize, (usize, usize))> {
385    parser.trim();
386
387    // Returns (operator type, length of operator, left and right precedence power)
388    match parser.next() {
389        Ok('|') if parser.peek_early(1, 1) == Ok("|") => Some((BinaryOp::BoolOr, 2, (8, 9))),
390        Ok('&') if parser.peek_early(1, 1) == Ok("&") => Some((BinaryOp::BoolAnd, 2, (10, 11))),
391        Ok('=') if parser.peek_early(1, 1) == Ok("=") => Some((BinaryOp::Equal, 2, (12, 13))),
392        Ok('!') if parser.peek_early(1, 1) == Ok("=") => Some((BinaryOp::NotEqual, 2, (12, 13))),
393        Ok('<') if parser.peek_early(1, 1) == Ok("=") => Some((BinaryOp::LessOrEqual, 2, (14, 15))),
394        Ok('>') if parser.peek_early(1, 1) == Ok("=") => {
395            Some((BinaryOp::GreaterOrEqual, 2, (14, 15)))
396        }
397        Ok('<') => Some((BinaryOp::Less, 1, (14, 15))),
398        Ok('>') => Some((BinaryOp::Greater, 1, (14, 15))),
399        Ok('.') if parser.peek_early(1, 1) == Ok(".") && parser.peek_early(1, 2) == Ok("=") => {
400            Some((BinaryOp::RangeInclusive, 3, (16, 17)))
401        }
402        Ok('.') if parser.peek_early(1, 1) == Ok(".") => Some((BinaryOp::Range, 2, (16, 17))),
403        Ok('+') => Some((BinaryOp::Add, 1, (18, 19))),
404        Ok('-') => Some((BinaryOp::Subtract, 1, (18, 19))),
405        Ok('*') => Some((BinaryOp::Multiply, 1, (20, 21))),
406        Ok('/') => Some((BinaryOp::Divide, 1, (20, 21))),
407
408        Ok(';') => Some((BinaryOp::Semicolon, 1, (1, 2))),
409        Ok('=') => Some((BinaryOp::Definition, 1, (3, 4))),
410
411        // The pipe operator has high right precedence but low left precedence
412        Ok('|') if parser.peek_early(1, 1) == Ok(">") => Some((BinaryOp::Pipe, 2, (5, 22))),
413        _ => None,
414    }
415}
416
417fn parse_ternary_operator(parser: &mut StreamParser) -> Option<(TernaryOp, usize, (usize, usize))> {
418    parser.trim();
419
420    // Returns (operator type, length of operator, left and right precedence power)
421    match parser.next() {
422        Ok('?') => Some((TernaryOp::Conditional, 1, (7, 6))),
423        _ => None,
424    }
425}
426
427fn string<'a>(parser: &mut StreamParser<'a>, file_offset: usize) -> Result<Literal<'a>> {
428    let index = file_offset + parser.index();
429    if parser.take_exact("\"").is_ok() {
430        let mut is_escaped = false;
431        let result = parser
432            .take_while(|(_, ch)| {
433                if is_escaped {
434                    is_escaped = false;
435                    true
436                } else if ch == '\\' {
437                    is_escaped = true;
438                    true
439                } else {
440                    ch != '"'
441                }
442            })
443            .to_string();
444        parser
445            .take_exact("\"")
446            .auto_error_offset(file_offset)
447            .map_err(|e| match *e {
448                Error::StreamParserError(pochoir_common::Error::UnexpectedEoi) => {
449                    Spanned::new(Error::UnterminatedString)
450                        .with_span(index..index + 1)
451                        .with_file_path(parser.file_path())
452                }
453                _ => e,
454            })?;
455
456        Ok(Literal::String(
457            result.replace("\\\"", "\"").replace("\\\\", "\\"),
458        ))
459    } else if parser.take_exact("'").is_ok() {
460        let mut is_escaped = false;
461        let result = parser
462            .take_while(|(_, ch)| {
463                if is_escaped {
464                    is_escaped = false;
465                    true
466                } else if ch == '\\' {
467                    is_escaped = true;
468                    true
469                } else {
470                    ch != '\''
471                }
472            })
473            .to_string();
474        parser
475            .take_exact("'")
476            .auto_error_offset(file_offset)
477            .map_err(|e| match *e {
478                Error::StreamParserError(pochoir_common::Error::UnexpectedEoi) => {
479                    Spanned::new(Error::UnterminatedString)
480                        .with_span(index..index + 1)
481                        .with_file_path(parser.file_path())
482                }
483                _ => e,
484            })?;
485
486        Ok(Literal::String(
487            result.replace("\\'", "'").replace("\\\\", "\\"),
488        ))
489    } else {
490        // String is called only when the parser starts with a " or '
491        unreachable!();
492    }
493}
494
495fn number<'a>(parser: &mut StreamParser<'a>) -> Result<Literal<'a>> {
496    // Avoid parsing ranges as numbers
497    let result = parser.take_while_peekable(|_, ch, next_ch| {
498        ch.is_numeric() || (ch == '.' && next_ch != Some('.'))
499    });
500
501    Ok(Literal::Number(
502        result.parse::<f64>().map_err(|_e| Error::InvalidNumber)?,
503    ))
504}
505
506fn array<'a>(parser: &mut StreamParser<'a>, file_offset: usize) -> Result<Literal<'a>> {
507    let index = file_offset + parser.index();
508    parser.take_exact("[").auto_error_offset(file_offset)?;
509
510    let mut values = vec![];
511
512    loop {
513        parser.trim();
514
515        if parser.peek_exact("]") {
516            break;
517        }
518
519        let mut field_values = vec![];
520        parse_recursive(parser, &mut field_values, 0, file_offset).map_err(|e| match *e {
521            Error::StreamParserError(pochoir_common::Error::UnexpectedEoi) => {
522                Spanned::new(Error::UnterminatedArray)
523                    .with_span(index..index + 1)
524                    .with_file_path(parser.file_path())
525            }
526            _ => e,
527        })?;
528        values.push(field_values);
529
530        if parser.take_exact(",").is_err() {
531            break;
532        }
533    }
534
535    parser.trim();
536    parser.take_exact("]").map_err(|_e| {
537        Spanned::new(Error::UnterminatedArray)
538            .with_span(index..index + 1)
539            .with_file_path(parser.file_path())
540    })?;
541    Ok(Literal::Array(values))
542}
543
544fn object<'a>(parser: &mut StreamParser<'a>, file_offset: usize) -> Result<Literal<'a>> {
545    let index = file_offset + parser.index();
546    parser.take_exact("{").auto_error_offset(file_offset)?;
547
548    let mut values = IndexMap::new();
549
550    loop {
551        parser.trim();
552
553        if parser.peek_exact("}") {
554            break;
555        }
556
557        let key = if parser.peek_exact("\"") || parser.peek_exact("'") {
558            if let Literal::String(string) = string(parser, file_offset)? {
559                string
560            } else {
561                unreachable!();
562            }
563        } else {
564            let mut first = true;
565            parser
566                .take_while(|(_, ch)| {
567                    if first {
568                        first = false;
569                        ch.is_alphabetic() || ch == '_'
570                    } else {
571                        ch.is_alphanumeric() || ch == '_'
572                    }
573                })
574                .trim()
575                .to_string()
576        };
577        parser
578            .take_exact(":")
579            .auto_error_offset(file_offset)
580            .map_err(|e| match *e {
581                Error::StreamParserError(pochoir_common::Error::UnexpectedEoi) => {
582                    Spanned::new(Error::UnterminatedObject)
583                        .with_span(index..index + 1)
584                        .with_file_path(parser.file_path())
585                }
586                _ => e,
587            })?;
588        parser.trim();
589
590        let mut field_values = vec![];
591        parse_recursive(parser, &mut field_values, 0, file_offset).map_err(|e| match *e {
592            Error::StreamParserError(pochoir_common::Error::UnexpectedEoi) => {
593                Spanned::new(Error::UnterminatedObject)
594                    .with_span(index..index + 1)
595                    .with_file_path(parser.file_path())
596            }
597            _ => e,
598        })?;
599        values.insert(key, field_values);
600
601        if parser.take_exact(",").is_err() {
602            break;
603        }
604    }
605
606    parser.trim();
607    parser.take_exact("}").map_err(|_e| {
608        Spanned::new(Error::UnterminatedObject)
609            .with_span(index..index + 1)
610            .with_file_path(parser.file_path())
611    })?;
612    Ok(Literal::Object(values))
613}
614
615#[cfg(test)]
616mod tests {
617    use super::*;
618    use indexmap::indexmap;
619    use pretty_assertions::assert_eq;
620
621    #[test]
622    fn parse_number() {
623        assert_eq!(
624            parse("index.html", "42", 0),
625            Ok(vec![Spanned::new(Token::Literal(Literal::Number(42.0)))
626                .with_span(0..2)
627                .with_file_path("index.html")])
628        );
629    }
630
631    #[test]
632    fn mathematical_priority_test() {
633        assert_eq!(
634            parse("index.html", "6/ 2 *(2+1) == 9", 0),
635            Ok(vec![Spanned::new(Token::BinaryOperator(
636                BinaryOp::Equal,
637                vec![Spanned::new(Token::BinaryOperator(
638                    BinaryOp::Multiply,
639                    vec![Spanned::new(Token::BinaryOperator(
640                        BinaryOp::Divide,
641                        vec![Spanned::new(Token::Literal(Literal::Number(6.0)))
642                            .with_span(0..1)
643                            .with_file_path("index.html")],
644                        vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
645                            .with_span(3..4)
646                            .with_file_path("index.html")],
647                    ))
648                    .with_span(0..4)
649                    .with_file_path("index.html")],
650                    vec![Spanned::new(Token::BinaryOperator(
651                        BinaryOp::Add,
652                        vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
653                            .with_span(7..8)
654                            .with_file_path("index.html")],
655                        vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
656                            .with_span(9..10)
657                            .with_file_path("index.html")],
658                    ))
659                    .with_span(7..10)
660                    .with_file_path("index.html")],
661                ))
662                .with_span(0..11)
663                .with_file_path("index.html")],
664                vec![Spanned::new(Token::Literal(Literal::Number(9.0)))
665                    .with_span(15..16)
666                    .with_file_path("index.html")]
667            ))
668            .with_span(0..16)
669            .with_file_path("index.html")]),
670        );
671    }
672
673    #[test]
674    fn array_indexing() {
675        assert_eq!(
676            parse("index.html", "my_arr[1] == 2.0", 0),
677            Ok(vec![Spanned::new(Token::BinaryOperator(
678                BinaryOp::Equal,
679                vec![Spanned::new(Token::Index(
680                    vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
681                        .with_span(7..8)
682                        .with_file_path("index.html")],
683                    Box::new(
684                        Spanned::new(Token::Variable(Cow::Borrowed("my_arr")))
685                            .with_span(0..6)
686                            .with_file_path("index.html")
687                    ),
688                ))
689                .with_span(0..9)
690                .with_file_path("index.html")],
691                vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
692                    .with_span(13..16)
693                    .with_file_path("index.html")]
694            ))
695            .with_span(0..16)
696            .with_file_path("index.html")]),
697        );
698    }
699
700    #[test]
701    fn function_args_newline() {
702        assert_eq!(
703            parse(
704                "index.html",
705                r#"test(
706    "a",
707    42
708)"#,
709                0
710            ),
711            Ok(vec![Spanned::new(Token::FunctionCall(
712                Box::new(
713                    Spanned::new(Token::Variable(Cow::Borrowed("test")))
714                        .with_span(0..4)
715                        .with_file_path("index.html")
716                ),
717                vec![
718                    vec![
719                        Spanned::new(Token::Literal(Literal::String("a".to_string())))
720                            .with_span(10..13)
721                            .with_file_path("index.html")
722                    ],
723                    vec![Spanned::new(Token::Literal(Literal::Number(42.0)))
724                        .with_span(19..21)
725                        .with_file_path("index.html")],
726                ]
727            ))
728            .with_span(0..23)
729            .with_file_path("index.html")])
730        );
731    }
732
733    #[test]
734    fn function_args_trailing_comma() {
735        assert_eq!(
736            parse(
737                "index.html",
738                r#"test(
739    "a",
740    42,
741)"#,
742                0
743            ),
744            Ok(vec![Spanned::new(Token::FunctionCall(
745                Box::new(
746                    Spanned::new(Token::Variable(Cow::Borrowed("test")))
747                        .with_span(0..4)
748                        .with_file_path("index.html")
749                ),
750                vec![
751                    vec![
752                        Spanned::new(Token::Literal(Literal::String("a".to_string())))
753                            .with_span(10..13)
754                            .with_file_path("index.html")
755                    ],
756                    vec![Spanned::new(Token::Literal(Literal::Number(42.0)))
757                        .with_span(19..21)
758                        .with_file_path("index.html")],
759                ]
760            ))
761            .with_span(0..24)
762            .with_file_path("index.html")])
763        );
764
765        assert_eq!(
766            parse("index.html", r#"test("a", 42,)"#, 0),
767            Ok(vec![Spanned::new(Token::FunctionCall(
768                Box::new(
769                    Spanned::new(Token::Variable(Cow::Borrowed("test")))
770                        .with_span(0..4)
771                        .with_file_path("index.html")
772                ),
773                vec![
774                    vec![
775                        Spanned::new(Token::Literal(Literal::String("a".to_string())))
776                            .with_span(5..8)
777                            .with_file_path("index.html")
778                    ],
779                    vec![Spanned::new(Token::Literal(Literal::Number(42.0)))
780                        .with_span(10..12)
781                        .with_file_path("index.html")],
782                ]
783            ))
784            .with_span(0..14)
785            .with_file_path("index.html")])
786        );
787    }
788
789    #[test]
790    fn namespaced_function() {
791        assert_eq!(
792            parse(
793                "index.html",
794                "Restaurants.get('Killer Pizza from Mars').location == 'Oceanside, CA'",
795                0
796            ),
797            Ok(vec![Spanned::new(Token::BinaryOperator(
798                BinaryOp::Equal,
799                vec![Spanned::new(Token::Index(
800                    vec![
801                        Spanned::new(Token::Literal(Literal::String("location".to_string())))
802                            .with_span(42..50)
803                            .with_file_path("index.html")
804                    ],
805                    Box::new(
806                        Spanned::new(Token::FunctionCall(
807                            Box::new(
808                                Spanned::new(Token::Index(
809                                    vec![Spanned::new(Token::Literal(Literal::String(
810                                        "get".to_string()
811                                    )))
812                                    .with_span(12..15)
813                                    .with_file_path("index.html")],
814                                    Box::new(
815                                        Spanned::new(Token::Variable(Cow::Borrowed("Restaurants")))
816                                            .with_span(0..11)
817                                            .with_file_path("index.html")
818                                    )
819                                ))
820                                .with_span(0..15)
821                                .with_file_path("index.html")
822                            ),
823                            vec![vec![Spanned::new(Token::Literal(Literal::String(
824                                "Killer Pizza from Mars".to_string()
825                            )))
826                            .with_span(16..40)
827                            .with_file_path("index.html")]],
828                        ))
829                        .with_span(0..41)
830                        .with_file_path("index.html")
831                    )
832                ))
833                .with_span(0..50)
834                .with_file_path("index.html")],
835                vec![
836                    Spanned::new(Token::Literal(Literal::String("Oceanside, CA".to_string())))
837                        .with_span(54..69)
838                        .with_file_path("index.html")
839                ],
840            ))
841            .with_span(0..69)
842            .with_file_path("index.html")]),
843        );
844    }
845
846    #[test]
847    fn object_newline() {
848        assert_eq!(
849            parse(
850                "index.html",
851                r#"{
852                    "a": "b",
853                    "c": 42,
854                }"#,
855                0
856            ),
857            Ok(vec![Spanned::new(Token::Literal(Literal::Object(
858                indexmap! {
859                    String::from("a") => vec![Spanned::new(Token::Literal(Literal::String("b".to_string())))
860                        .with_span(27..30)
861                        .with_file_path("index.html")
862                    ],
863                    String::from("c") => vec![Spanned::new(Token::Literal(Literal::Number(42.0)))
864                        .with_span(57..59)
865                        .with_file_path("index.html")
866                    ],
867                }
868            )))
869            .with_span(0..78)
870            .with_file_path("index.html")])
871        );
872    }
873
874    #[test]
875    fn array_newline() {
876        assert_eq!(
877            parse(
878                "index.html",
879                r#"[
880                    "b",
881                    42,
882                ]"#,
883                0
884            ),
885            Ok(vec![Spanned::new(Token::Literal(Literal::Array(vec![
886                vec![
887                    Spanned::new(Token::Literal(Literal::String("b".to_string())))
888                        .with_span(22..25)
889                        .with_file_path("index.html")
890                ],
891                vec![Spanned::new(Token::Literal(Literal::Number(42.0)))
892                    .with_span(47..49)
893                    .with_file_path("index.html")],
894            ])))
895            .with_span(0..68)
896            .with_file_path("index.html")])
897        );
898    }
899
900    #[test]
901    #[allow(clippy::too_many_lines)]
902    fn range() {
903        assert_eq!(
904            parse("index.html", "2..3", 0),
905            Ok(vec![Spanned::new(Token::BinaryOperator(
906                BinaryOp::Range,
907                vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
908                    .with_span(0..1)
909                    .with_file_path("index.html")],
910                vec![Spanned::new(Token::Literal(Literal::Number(3.0)))
911                    .with_span(3..4)
912                    .with_file_path("index.html")],
913            ))
914            .with_span(0..4)
915            .with_file_path("index.html")]),
916        );
917
918        assert_eq!(
919            parse("index.html", "3..2", 0),
920            Ok(vec![Spanned::new(Token::BinaryOperator(
921                BinaryOp::Range,
922                vec![Spanned::new(Token::Literal(Literal::Number(3.0)))
923                    .with_span(0..1)
924                    .with_file_path("index.html")],
925                vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
926                    .with_span(3..4)
927                    .with_file_path("index.html")],
928            ))
929            .with_span(0..4)
930            .with_file_path("index.html")]),
931        );
932
933        assert_eq!(
934            parse("index.html", "..3", 0),
935            Ok(vec![Spanned::new(Token::UnaryOperator(
936                UnaryOp::Range,
937                vec![Spanned::new(Token::Literal(Literal::Number(3.0)))
938                    .with_span(2..3)
939                    .with_file_path("index.html")],
940            ))
941            .with_span(0..3)
942            .with_file_path("index.html")]),
943        );
944
945        assert_eq!(
946            parse("index.html", "2..", 0),
947            Ok(vec![Spanned::new(Token::BinaryOperator(
948                BinaryOp::Range,
949                vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
950                    .with_span(0..1)
951                    .with_file_path("index.html")],
952                vec![],
953            ))
954            .with_span(0..3)
955            .with_file_path("index.html")]),
956        );
957
958        assert_eq!(
959            parse("index.html", "1 + 2..4", 0),
960            Ok(vec![Spanned::new(Token::BinaryOperator(
961                BinaryOp::Range,
962                vec![Spanned::new(Token::BinaryOperator(
963                    BinaryOp::Add,
964                    vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
965                        .with_span(0..1)
966                        .with_file_path("index.html")],
967                    vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
968                        .with_span(4..5)
969                        .with_file_path("index.html")],
970                ))
971                .with_span(0..5)
972                .with_file_path("index.html")],
973                vec![Spanned::new(Token::Literal(Literal::Number(4.0)))
974                    .with_span(7..8)
975                    .with_file_path("index.html")],
976            ))
977            .with_span(0..8)
978            .with_file_path("index.html")]),
979        );
980
981        assert_eq!(
982            parse("index.html", "1..1 + 1", 0),
983            Ok(vec![Spanned::new(Token::BinaryOperator(
984                BinaryOp::Range,
985                vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
986                    .with_span(0..1)
987                    .with_file_path("index.html")],
988                vec![Spanned::new(Token::BinaryOperator(
989                    BinaryOp::Add,
990                    vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
991                        .with_span(3..4)
992                        .with_file_path("index.html")],
993                    vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
994                        .with_span(7..8)
995                        .with_file_path("index.html")],
996                ))
997                .with_span(3..8)
998                .with_file_path("index.html")],
999            ))
1000            .with_span(0..8)
1001            .with_file_path("index.html")]),
1002        );
1003
1004        assert_eq!(
1005            parse("index.html", r#"len("a")..3"#, 0),
1006            Ok(vec![Spanned::new(Token::BinaryOperator(
1007                BinaryOp::Range,
1008                vec![Spanned::new(Token::FunctionCall(
1009                    Box::new(
1010                        Spanned::new(Token::Variable(Cow::Borrowed("len")))
1011                            .with_span(0..3)
1012                            .with_file_path("index.html")
1013                    ),
1014                    vec![vec![Spanned::new(Token::Literal(Literal::String(
1015                        "a".to_string()
1016                    )))
1017                    .with_span(4..7)
1018                    .with_file_path("index.html")]]
1019                ))
1020                .with_span(0..8)
1021                .with_file_path("index.html")],
1022                vec![Spanned::new(Token::Literal(Literal::Number(3.0)))
1023                    .with_span(10..11)
1024                    .with_file_path("index.html")],
1025            ))
1026            .with_span(0..11)
1027            .with_file_path("index.html")]),
1028        );
1029
1030        assert_eq!(
1031            parse("index.html", r#""a"..3"#, 0),
1032            Ok(vec![Spanned::new(Token::BinaryOperator(
1033                BinaryOp::Range,
1034                vec![
1035                    Spanned::new(Token::Literal(Literal::String("a".to_string())))
1036                        .with_span(0..3)
1037                        .with_file_path("index.html")
1038                ],
1039                vec![Spanned::new(Token::Literal(Literal::Number(3.0)))
1040                    .with_span(5..6)
1041                    .with_file_path("index.html")],
1042            ))
1043            .with_span(0..6)
1044            .with_file_path("index.html")]),
1045        );
1046
1047        assert_eq!(
1048            parse("index.html", "1..3[2]", 0),
1049            Ok(vec![Spanned::new(Token::BinaryOperator(
1050                BinaryOp::Range,
1051                vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
1052                    .with_span(0..1)
1053                    .with_file_path("index.html")],
1054                vec![Spanned::new(Token::Index(
1055                    vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
1056                        .with_span(5..6)
1057                        .with_file_path("index.html"),],
1058                    Box::new(
1059                        Spanned::new(Token::Literal(Literal::Number(3.0)))
1060                            .with_span(3..4)
1061                            .with_file_path("index.html")
1062                    )
1063                ))
1064                .with_span(3..7)
1065                .with_file_path("index.html")]
1066            ))
1067            .with_span(0..7)
1068            .with_file_path("index.html")]),
1069        );
1070
1071        assert_eq!(
1072            parse("index.html", "1..3.2", 0),
1073            Ok(vec![Spanned::new(Token::BinaryOperator(
1074                BinaryOp::Range,
1075                vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
1076                    .with_span(0..1)
1077                    .with_file_path("index.html")],
1078                vec![Spanned::new(Token::Literal(Literal::Number(3.2)))
1079                    .with_span(3..6)
1080                    .with_file_path("index.html")],
1081            ))
1082            .with_span(0..6)
1083            .with_file_path("index.html")]),
1084        );
1085
1086        assert_eq!(
1087            parse("index.html", "0..2 == ..=1", 0),
1088            Ok(vec![Spanned::new(Token::BinaryOperator(
1089                BinaryOp::Equal,
1090                vec![Spanned::new(Token::BinaryOperator(
1091                    BinaryOp::Range,
1092                    vec![Spanned::new(Token::Literal(Literal::Number(0.0)))
1093                        .with_span(0..1)
1094                        .with_file_path("index.html")],
1095                    vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
1096                        .with_span(3..4)
1097                        .with_file_path("index.html")],
1098                ))
1099                .with_span(0..4)
1100                .with_file_path("index.html")],
1101                vec![Spanned::new(Token::UnaryOperator(
1102                    UnaryOp::RangeInclusive,
1103                    vec![Spanned::new(Token::Literal(Literal::Number(1.0)))
1104                        .with_span(11..12)
1105                        .with_file_path("index.html")],
1106                ))
1107                .with_span(8..12)
1108                .with_file_path("index.html")],
1109            ))
1110            .with_span(0..12)
1111            .with_file_path("index.html")]),
1112        );
1113
1114        assert_eq!(
1115            parse("index.html", "my_fn(2..)", 0),
1116            Ok(vec![Spanned::new(Token::FunctionCall(
1117                Box::new(
1118                    Spanned::new(Token::Variable(Cow::Borrowed("my_fn")))
1119                        .with_span(0..5)
1120                        .with_file_path("index.html")
1121                ),
1122                vec![vec![Spanned::new(Token::BinaryOperator(
1123                    BinaryOp::Range,
1124                    vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
1125                        .with_span(6..7)
1126                        .with_file_path("index.html")],
1127                    vec![],
1128                ))
1129                .with_span(6..9)
1130                .with_file_path("index.html")]],
1131            ))
1132            .with_span(0..10)
1133            .with_file_path("index.html")]),
1134        );
1135
1136        assert_eq!(
1137            parse("index.html", r#""hello"[2..=4] == "ll""#, 0),
1138            Ok(vec![Spanned::new(Token::BinaryOperator(
1139                BinaryOp::Equal,
1140                vec![Spanned::new(Token::Index(
1141                    vec![Spanned::new(Token::BinaryOperator(
1142                        BinaryOp::RangeInclusive,
1143                        vec![Spanned::new(Token::Literal(Literal::Number(2.0)))
1144                            .with_span(8..9)
1145                            .with_file_path("index.html")],
1146                        vec![Spanned::new(Token::Literal(Literal::Number(4.0)))
1147                            .with_span(12..13)
1148                            .with_file_path("index.html")],
1149                    ))
1150                    .with_span(8..13)
1151                    .with_file_path("index.html")],
1152                    Box::new(
1153                        Spanned::new(Token::Literal(Literal::String("hello".to_string())))
1154                            .with_span(0..7)
1155                            .with_file_path("index.html")
1156                    )
1157                ))
1158                .with_span(0..14)
1159                .with_file_path("index.html")],
1160                vec![
1161                    Spanned::new(Token::Literal(Literal::String("ll".to_string())))
1162                        .with_span(18..22)
1163                        .with_file_path("index.html")
1164                ],
1165            ))
1166            .with_span(0..22)
1167            .with_file_path("index.html")]),
1168        );
1169    }
1170}
pochoir_lang/parser.rs

pochoir_lang/
parser.rs
