yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser.
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::syntax::HereDoc;
31use crate::syntax::MaybeLiteral;
32use std::rc::Rc;
33
34/// Entire result of parsing.
35pub type Result<T> = std::result::Result<T, Error>;
36
37/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
38/// parsing after alias substitution.
39///
40/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
41///
42/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
43/// meaningful value and is returned from a parsing function that has performed alias substitution
44/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
45/// new source code produced by the substitution so that the syntax is correctly recognized in the
46/// new code.
47///
48/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
49/// recognized as an alias name during parse of a simple command, the simple command parser
50/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
51/// parser function, to recognize the `!` reserved word token as negation.
52///
53/// When a parser function successfully parses something, it returns the result in the `Parsed`
54/// variant. The caller then continues the remaining parse.
55#[derive(Copy, Clone, Debug, Eq, PartialEq)]
56pub enum Rec<T> {
57    /// Result of alias substitution.
58    AliasSubstituted,
59    /// Successful parse result.
60    Parsed(T),
61}
62
63impl<T> Rec<T> {
64    /// Tests if `self` is `AliasSubstituted`.
65    pub fn is_alias_substituted(&self) -> bool {
66        match self {
67            Rec::AliasSubstituted => true,
68            Rec::Parsed(_) => false,
69        }
70    }
71
72    /// Extracts the result of successful parsing.
73    ///
74    /// # Panics
75    ///
76    /// If `self` is `AliasSubstituted`.
77    pub fn unwrap(self) -> T {
78        match self {
79            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
80            Rec::Parsed(v) => v,
81        }
82    }
83
84    /// Transforms the result value in `self`.
85    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
86    where
87        F: FnOnce(T) -> Result<U>,
88    {
89        match self {
90            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
91            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
92        }
93    }
94}
95
96/// The shell syntax parser.
97///
98/// This `struct` contains a set of data used in syntax parsing.
99///
100/// # Parsing here-documents
101///
102/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
103/// with empty content. The parser creates the `HereDoc` instance when it finds
104/// a here-document operator, but it has not read its content at that time. When
105/// finding a newline token, the parser reads the content and fills it into the
106/// `HereDoc` instance.
107///
108/// Unless you are interested in parsing a specific syntactic construct that is
109/// only part of source code, you will want to use a function that returns a
110/// complete result filled with proper here-document contents if any.
111/// Then the [`command_line`](Self::command_line) function is for you.
112/// See also the [module documentation](super).
113#[derive(Debug)]
114pub struct Parser<'a, 'b> {
115    /// Lexer that provides tokens.
116    lexer: &'a mut Lexer<'b>,
117
118    /// Collection of aliases the parser applies to substitute command words.
119    glossary: &'a dyn Glossary,
120
121    /// Token to parse next.
122    ///
123    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
124    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
125    token: Option<Result<Token>>,
126
127    /// Here-documents without contents.
128    ///
129    /// The here-document is added to this list when the parser finds a
130    /// here-document operator. After consuming the next newline token, the
131    /// parser reads and fills the contents, then clears this list.
132    unread_here_docs: Vec<Rc<HereDoc>>,
133}
134
135impl<'a, 'b> Parser<'a, 'b> {
136    /// Creates a new parser based on the given lexer and glossary.
137    ///
138    /// The parser uses the lexer to read tokens and the glossary to look up aliases.
139    pub fn new(lexer: &'a mut Lexer<'b>, glossary: &'a dyn Glossary) -> Parser<'a, 'b> {
140        Parser {
141            lexer,
142            glossary,
143            token: None,
144            unread_here_docs: vec![],
145        }
146    }
147
148    /// Reads a next token if the current token is `None`.
149    async fn require_token(&mut self) {
150        #[allow(clippy::question_mark)] // TODO https://github.com/rust-lang/rust-clippy/issues/9518
151        if self.token.is_none() {
152            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
153                Err(e)
154            } else {
155                self.lexer.token().await
156            });
157        }
158    }
159
160    /// Returns a reference to the current token.
161    ///
162    /// If the current token is not yet read from the underlying lexer, it is read.
163    pub async fn peek_token(&mut self) -> Result<&Token> {
164        self.require_token().await;
165        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
166    }
167
168    /// Consumes the current token without performing alias substitution.
169    ///
170    /// If the current token is not yet read from the underlying lexer, it is read.
171    ///
172    /// This function does not perform alias substitution and therefore should be
173    /// used only in context where no alias substitution is expected. Otherwise,
174    /// you should use [`take_token_manual`](Self::take_token_manual) or
175    /// [`take_token_auto`](Self::take_token_auto) instead.
176    pub async fn take_token_raw(&mut self) -> Result<Token> {
177        self.require_token().await;
178        self.token.take().unwrap()
179    }
180
181    /// Performs alias substitution on a token that has just been
182    /// [taken](Self::take_token_raw).
183    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
184        // TODO Only POSIXly-valid alias name should be recognized in POSIXly-correct mode.
185        if !self.glossary.is_empty() {
186            if let Token(_) = token.id {
187                if let Some(name) = token.word.to_string_if_literal() {
188                    if !token.word.location.code.source.is_alias_for(&name) {
189                        if let Some(alias) = self.glossary.look_up(&name) {
190                            if is_command_name
191                                || alias.global
192                                || self.lexer.is_after_blank_ending_alias(token.index)
193                            {
194                                self.lexer.substitute_alias(token.index, &alias);
195                                return Rec::AliasSubstituted;
196                            }
197                        }
198                    }
199                }
200            }
201        }
202
203        Rec::Parsed(token)
204    }
205
206    /// Consumes the current token after performing applicable alias substitution.
207    ///
208    /// If the current token is not yet read from the underlying lexer, it is read.
209    ///
210    /// This function checks if the token is the name of an alias. If it is,
211    /// alias substitution is performed on the token and the result is
212    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
213    ///
214    /// Alias substitution is performed only if at least one of the following is
215    /// true:
216    ///
217    /// - The token is the first command word in a simple command, that is, it is
218    ///   the word for the command name. (This condition should be specified by the
219    ///   `is_command_name` parameter.)
220    /// - The token comes just after the replacement string of another alias
221    ///   substitution that ends with a blank character.
222    /// - The token names a global alias.
223    ///
224    /// However, alias substitution should _not_ be performed on a reserved word
225    /// in any case. It is your responsibility to check the token type and not to
226    /// call this function on a reserved word. That is why this function is named
227    /// `manual`. To consume a reserved word without performing alias
228    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
229    /// [`take_token_auto`](Self::take_token_auto).
230    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
231        let token = self.take_token_raw().await?;
232        Ok(self.substitute_alias(token, is_command_name))
233    }
234
235    /// Consumes the current token after performing applicable alias substitution.
236    ///
237    /// This function performs alias substitution unless the result is one of the
238    /// reserved words specified in the argument.
239    ///
240    /// Alias substitution is performed repeatedly until a non-alias token is
241    /// found. That is why this function is named `auto`. This function should be
242    /// used only in contexts where no backtrack is needed after alias
243    /// substitution. If you need to backtrack or want to know whether alias
244    /// substitution was performed or not, you should use
245    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
246    /// alias substitution at most once and returns `Rec`.
247    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
248        loop {
249            let token = self.take_token_raw().await?;
250            if let Token(Some(keyword)) = token.id {
251                if keywords.contains(&keyword) {
252                    return Ok(token);
253                }
254            }
255            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
256                return Ok(token);
257            }
258        }
259    }
260
261    /// Tests if there is a blank before the next token.
262    ///
263    /// This function can be called to tell whether the previous and next tokens
264    /// are separated by a blank or they are adjacent.
265    ///
266    /// This function must be called after the previous token has been taken (by
267    /// one of [`take_token_raw`](Self::take_token_raw),
268    /// [`take_token_manual`](Self::take_token_manual) and
269    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
270    /// [peeked](Self::peek_token). Otherwise, this function would panic.
271    ///
272    /// # Panics
273    ///
274    /// If the previous token has not been taken or the next token has been
275    /// peeked.
276    pub async fn has_blank(&mut self) -> Result<bool> {
277        assert!(self.token.is_none(), "There should be no pending token");
278        let c = self.lexer.peek_char().await?;
279        Ok(c.map_or(false, is_blank))
280    }
281
282    /// Remembers the given partial here-document for later parsing of its content.
283    ///
284    /// The remembered here-document's content will be parsed when
285    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
286    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
287        self.unread_here_docs.push(here_doc)
288    }
289
290    /// Reads here-document contents that matches the remembered list of
291    /// here-document operators.
292    ///
293    /// This function reads here-document contents corresponding to
294    /// here-document operators that have been saved with
295    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
296    /// The results are inserted to the `content` field of the `HereDoc`
297    /// instances.
298    ///
299    /// This function must be called just after a newline token has been taken
300    /// (either [manual](Self::take_token_manual) or
301    /// [auto](Self::take_token_auto)). If there is a pending token that has been
302    /// peeked but not yet taken, this function will panic!
303    pub async fn here_doc_contents(&mut self) -> Result<()> {
304        assert!(
305            self.token.is_none(),
306            "No token must be peeked before reading here-doc contents"
307        );
308
309        for here_doc in self.unread_here_docs.drain(..) {
310            self.lexer.here_doc_content(&here_doc).await?;
311        }
312
313        Ok(())
314    }
315
316    /// Ensures that there is no pending partial here-document.
317    ///
318    /// If there is any, this function returns a `MissingHereDocContent` error.
319    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
320        match self.unread_here_docs.first() {
321            None => Ok(()),
322            Some(here_doc) => Err(Error {
323                cause: SyntaxError::MissingHereDocContent.into(),
324                location: here_doc.delimiter.location.clone(),
325            }),
326        }
327    }
328}
329
330#[allow(clippy::bool_assert_comparison)]
331#[cfg(test)]
332mod tests {
333    use super::*;
334    use crate::alias::AliasSet;
335    use crate::alias::HashEntry;
336    use crate::source::Location;
337    use crate::source::Source;
338    use assert_matches::assert_matches;
339    use futures_util::FutureExt;
340    use std::cell::OnceCell;
341
342    #[test]
343    fn parser_take_token_manual_successful_substitution() {
344        let mut lexer = Lexer::from_memory("X", Source::Unknown);
345        #[allow(clippy::mutable_key_type)]
346        let mut aliases = AliasSet::new();
347        aliases.insert(HashEntry::new(
348            "X".to_string(),
349            "x".to_string(),
350            false,
351            Location::dummy("?"),
352        ));
353        let mut parser = Parser::new(&mut lexer, &aliases);
354
355        let result = parser.take_token_manual(true).now_or_never().unwrap();
356        assert_matches!(result, Ok(Rec::AliasSubstituted));
357
358        let result = parser.take_token_manual(true).now_or_never().unwrap();
359        let token = result.unwrap().unwrap();
360        assert_eq!(token.to_string(), "x");
361    }
362
363    #[test]
364    fn parser_take_token_manual_not_command_name() {
365        let mut lexer = Lexer::from_memory("X", Source::Unknown);
366        #[allow(clippy::mutable_key_type)]
367        let mut aliases = AliasSet::new();
368        aliases.insert(HashEntry::new(
369            "X".to_string(),
370            "x".to_string(),
371            false,
372            Location::dummy("?"),
373        ));
374        let mut parser = Parser::new(&mut lexer, &aliases);
375
376        let result = parser.take_token_manual(false).now_or_never().unwrap();
377        let token = result.unwrap().unwrap();
378        assert_eq!(token.to_string(), "X");
379    }
380
381    #[test]
382    fn parser_take_token_manual_not_literal() {
383        let mut lexer = Lexer::from_memory(r"\X", Source::Unknown);
384        #[allow(clippy::mutable_key_type)]
385        let mut aliases = AliasSet::new();
386        aliases.insert(HashEntry::new(
387            "X".to_string(),
388            "x".to_string(),
389            false,
390            Location::dummy("?"),
391        ));
392        aliases.insert(HashEntry::new(
393            r"\X".to_string(),
394            "quoted".to_string(),
395            false,
396            Location::dummy("?"),
397        ));
398        let mut parser = Parser::new(&mut lexer, &aliases);
399
400        let result = parser.take_token_manual(true).now_or_never().unwrap();
401        let token = result.unwrap().unwrap();
402        assert_eq!(token.to_string(), r"\X");
403    }
404
405    #[test]
406    fn parser_take_token_manual_operator() {
407        let mut lexer = Lexer::from_memory(";", Source::Unknown);
408        #[allow(clippy::mutable_key_type)]
409        let mut aliases = AliasSet::new();
410        aliases.insert(HashEntry::new(
411            ";".to_string(),
412            "x".to_string(),
413            false,
414            Location::dummy("?"),
415        ));
416        let mut parser = Parser::new(&mut lexer, &aliases);
417
418        let result = parser.take_token_manual(true).now_or_never().unwrap();
419        let token = result.unwrap().unwrap();
420        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
421        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
422    }
423
424    #[test]
425    fn parser_take_token_manual_no_match() {
426        let mut lexer = Lexer::from_memory("X", Source::Unknown);
427        #[allow(clippy::mutable_key_type)]
428        let aliases = AliasSet::new();
429        let mut parser = Parser::new(&mut lexer, &aliases);
430
431        let result = parser.take_token_manual(true).now_or_never().unwrap();
432        let token = result.unwrap().unwrap();
433        assert_eq!(token.to_string(), "X");
434    }
435
436    #[test]
437    fn parser_take_token_manual_recursive_substitution() {
438        let mut lexer = Lexer::from_memory("X", Source::Unknown);
439        #[allow(clippy::mutable_key_type)]
440        let mut aliases = AliasSet::new();
441        aliases.insert(HashEntry::new(
442            "X".to_string(),
443            "Y x".to_string(),
444            false,
445            Location::dummy("?"),
446        ));
447        aliases.insert(HashEntry::new(
448            "Y".to_string(),
449            "X y".to_string(),
450            false,
451            Location::dummy("?"),
452        ));
453        let mut parser = Parser::new(&mut lexer, &aliases);
454
455        let result = parser.take_token_manual(true).now_or_never().unwrap();
456        assert_matches!(result, Ok(Rec::AliasSubstituted));
457
458        let result = parser.take_token_manual(true).now_or_never().unwrap();
459        assert_matches!(result, Ok(Rec::AliasSubstituted));
460
461        let result = parser.take_token_manual(true).now_or_never().unwrap();
462        let token = result.unwrap().unwrap();
463        assert_eq!(token.to_string(), "X");
464
465        let result = parser.take_token_manual(true).now_or_never().unwrap();
466        let token = result.unwrap().unwrap();
467        assert_eq!(token.to_string(), "y");
468
469        let rec = parser.take_token_manual(true).now_or_never().unwrap();
470        let token = rec.unwrap().unwrap();
471        assert_eq!(token.to_string(), "x");
472    }
473
474    #[test]
475    fn parser_take_token_manual_after_blank_ending_substitution() {
476        let mut lexer = Lexer::from_memory("X\tY", Source::Unknown);
477        #[allow(clippy::mutable_key_type)]
478        let mut aliases = AliasSet::new();
479        aliases.insert(HashEntry::new(
480            "X".to_string(),
481            " X ".to_string(),
482            false,
483            Location::dummy("?"),
484        ));
485        aliases.insert(HashEntry::new(
486            "Y".to_string(),
487            "y".to_string(),
488            false,
489            Location::dummy("?"),
490        ));
491        let mut parser = Parser::new(&mut lexer, &aliases);
492
493        let result = parser.take_token_manual(true).now_or_never().unwrap();
494        assert_matches!(result, Ok(Rec::AliasSubstituted));
495
496        let result = parser.take_token_manual(true).now_or_never().unwrap();
497        let token = result.unwrap().unwrap();
498        assert_eq!(token.to_string(), "X");
499
500        let result = parser.take_token_manual(false).now_or_never().unwrap();
501        assert_matches!(result, Ok(Rec::AliasSubstituted));
502
503        let result = parser.take_token_manual(false).now_or_never().unwrap();
504        let token = result.unwrap().unwrap();
505        assert_eq!(token.to_string(), "y");
506    }
507
508    #[test]
509    fn parser_take_token_manual_not_after_blank_ending_substitution() {
510        let mut lexer = Lexer::from_memory("X\tY", Source::Unknown);
511        #[allow(clippy::mutable_key_type)]
512        let mut aliases = AliasSet::new();
513        aliases.insert(HashEntry::new(
514            "X".to_string(),
515            " X".to_string(),
516            false,
517            Location::dummy("?"),
518        ));
519        aliases.insert(HashEntry::new(
520            "Y".to_string(),
521            "y".to_string(),
522            false,
523            Location::dummy("?"),
524        ));
525        let mut parser = Parser::new(&mut lexer, &aliases);
526
527        let result = parser.take_token_manual(true).now_or_never().unwrap();
528        assert_matches!(result, Ok(Rec::AliasSubstituted));
529
530        let result = parser.take_token_manual(true).now_or_never().unwrap();
531        let token = result.unwrap().unwrap();
532        assert_eq!(token.to_string(), "X");
533
534        let result = parser.take_token_manual(false).now_or_never().unwrap();
535        let token = result.unwrap().unwrap();
536        assert_eq!(token.to_string(), "Y");
537    }
538
539    #[test]
540    fn parser_take_token_manual_global() {
541        let mut lexer = Lexer::from_memory("X", Source::Unknown);
542        #[allow(clippy::mutable_key_type)]
543        let mut aliases = AliasSet::new();
544        aliases.insert(HashEntry::new(
545            "X".to_string(),
546            "x".to_string(),
547            true,
548            Location::dummy("?"),
549        ));
550        let mut parser = Parser::new(&mut lexer, &aliases);
551
552        let result = parser.take_token_manual(false).now_or_never().unwrap();
553        assert_matches!(result, Ok(Rec::AliasSubstituted));
554
555        let result = parser.take_token_manual(false).now_or_never().unwrap();
556        let token = result.unwrap().unwrap();
557        assert_eq!(token.to_string(), "x");
558    }
559
560    #[test]
561    fn parser_take_token_auto_non_keyword() {
562        let mut lexer = Lexer::from_memory("X", Source::Unknown);
563        #[allow(clippy::mutable_key_type)]
564        let mut aliases = AliasSet::new();
565        aliases.insert(HashEntry::new(
566            "X".to_string(),
567            "x".to_string(),
568            true,
569            Location::dummy("?"),
570        ));
571        let mut parser = Parser::new(&mut lexer, &aliases);
572
573        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
574        assert_eq!(token.to_string(), "x");
575    }
576
577    #[test]
578    fn parser_take_token_auto_keyword_matched() {
579        let mut lexer = Lexer::from_memory("if", Source::Unknown);
580        #[allow(clippy::mutable_key_type)]
581        let mut aliases = AliasSet::new();
582        aliases.insert(HashEntry::new(
583            "if".to_string(),
584            "x".to_string(),
585            true,
586            Location::dummy("?"),
587        ));
588        let mut parser = Parser::new(&mut lexer, &aliases);
589
590        let token = parser
591            .take_token_auto(&[Keyword::If])
592            .now_or_never()
593            .unwrap()
594            .unwrap();
595        assert_eq!(token.to_string(), "if");
596    }
597
598    #[test]
599    fn parser_take_token_auto_keyword_unmatched() {
600        let mut lexer = Lexer::from_memory("if", Source::Unknown);
601        #[allow(clippy::mutable_key_type)]
602        let mut aliases = AliasSet::new();
603        aliases.insert(HashEntry::new(
604            "if".to_string(),
605            "x".to_string(),
606            true,
607            Location::dummy("?"),
608        ));
609        let mut parser = Parser::new(&mut lexer, &aliases);
610
611        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
612        assert_eq!(token.to_string(), "x");
613    }
614
615    #[test]
616    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
617        let mut lexer = Lexer::from_memory("X", Source::Unknown);
618        #[allow(clippy::mutable_key_type)]
619        let mut aliases = AliasSet::new();
620        aliases.insert(HashEntry::new(
621            "X".to_string(),
622            "if".to_string(),
623            true,
624            Location::dummy("?"),
625        ));
626        aliases.insert(HashEntry::new(
627            "if".to_string(),
628            "x".to_string(),
629            true,
630            Location::dummy("?"),
631        ));
632        let mut parser = Parser::new(&mut lexer, &aliases);
633
634        let token = parser
635            .take_token_auto(&[Keyword::If])
636            .now_or_never()
637            .unwrap()
638            .unwrap();
639        assert_eq!(token.to_string(), "if");
640    }
641
642    #[test]
643    fn parser_has_blank_true() {
644        let mut lexer = Lexer::from_memory(" ", Source::Unknown);
645        #[allow(clippy::mutable_key_type)]
646        let aliases = AliasSet::new();
647        let mut parser = Parser::new(&mut lexer, &aliases);
648        let result = parser.has_blank().now_or_never().unwrap();
649        assert_eq!(result, Ok(true));
650    }
651
652    #[test]
653    fn parser_has_blank_false() {
654        let mut lexer = Lexer::from_memory("(", Source::Unknown);
655        #[allow(clippy::mutable_key_type)]
656        let aliases = AliasSet::new();
657        let mut parser = Parser::new(&mut lexer, &aliases);
658        let result = parser.has_blank().now_or_never().unwrap();
659        assert_eq!(result, Ok(false));
660    }
661
662    #[test]
663    fn parser_has_blank_eof() {
664        let mut lexer = Lexer::from_memory("", Source::Unknown);
665        #[allow(clippy::mutable_key_type)]
666        let aliases = AliasSet::new();
667        let mut parser = Parser::new(&mut lexer, &aliases);
668        let result = parser.has_blank().now_or_never().unwrap();
669        assert_eq!(result, Ok(false));
670    }
671
672    #[test]
673    fn parser_has_blank_true_with_line_continuations() {
674        let mut lexer = Lexer::from_memory("\\\n\\\n ", Source::Unknown);
675        #[allow(clippy::mutable_key_type)]
676        let aliases = AliasSet::new();
677        let mut parser = Parser::new(&mut lexer, &aliases);
678        let result = parser.has_blank().now_or_never().unwrap();
679        assert_eq!(result, Ok(true));
680    }
681
682    #[test]
683    fn parser_has_blank_false_with_line_continuations() {
684        let mut lexer = Lexer::from_memory("\\\n\\\n\\\n(", Source::Unknown);
685        #[allow(clippy::mutable_key_type)]
686        let aliases = AliasSet::new();
687        let mut parser = Parser::new(&mut lexer, &aliases);
688        let result = parser.has_blank().now_or_never().unwrap();
689        assert_eq!(result, Ok(false));
690    }
691
692    #[test]
693    #[should_panic(expected = "There should be no pending token")]
694    fn parser_has_blank_with_pending_token() {
695        let mut lexer = Lexer::from_memory("foo", Source::Unknown);
696        #[allow(clippy::mutable_key_type)]
697        let aliases = AliasSet::new();
698        let mut parser = Parser::new(&mut lexer, &aliases);
699        parser.peek_token().now_or_never().unwrap().unwrap();
700        let _ = parser.has_blank().now_or_never().unwrap();
701    }
702
703    #[test]
704    fn parser_reading_no_here_doc_contents() {
705        let mut lexer = Lexer::from_memory("X", Source::Unknown);
706        #[allow(clippy::mutable_key_type)]
707        let aliases = AliasSet::new();
708        let mut parser = Parser::new(&mut lexer, &aliases);
709        parser.here_doc_contents().now_or_never().unwrap().unwrap();
710
711        let location = lexer.location().now_or_never().unwrap().unwrap();
712        assert_eq!(location.code.start_line_number.get(), 1);
713        assert_eq!(location.range, 0..1);
714    }
715
716    #[test]
717    fn parser_reading_one_here_doc_content() {
718        let delimiter = "END".parse().unwrap();
719
720        let mut lexer = Lexer::from_memory("END\nX", Source::Unknown);
721        #[allow(clippy::mutable_key_type)]
722        let aliases = AliasSet::new();
723        let mut parser = Parser::new(&mut lexer, &aliases);
724        let remove_tabs = false;
725        let here_doc = Rc::new(HereDoc {
726            delimiter,
727            remove_tabs,
728            content: OnceCell::new(),
729        });
730        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
731        parser.here_doc_contents().now_or_never().unwrap().unwrap();
732        assert_eq!(here_doc.delimiter.to_string(), "END");
733        assert_eq!(here_doc.remove_tabs, remove_tabs);
734        assert_eq!(here_doc.content.get().unwrap().0, []);
735
736        let location = lexer.location().now_or_never().unwrap().unwrap();
737        assert_eq!(location.code.start_line_number.get(), 1);
738        assert_eq!(location.range, 4..5);
739    }
740
741    #[test]
742    fn parser_reading_many_here_doc_contents() {
743        let delimiter1 = "ONE".parse().unwrap();
744        let delimiter2 = "TWO".parse().unwrap();
745        let delimiter3 = "THREE".parse().unwrap();
746
747        let mut lexer = Lexer::from_memory("1\nONE\nTWO\n3\nTHREE\nX", Source::Unknown);
748        #[allow(clippy::mutable_key_type)]
749        let aliases = AliasSet::new();
750        let mut parser = Parser::new(&mut lexer, &aliases);
751        let here_doc1 = Rc::new(HereDoc {
752            delimiter: delimiter1,
753            remove_tabs: false,
754            content: OnceCell::new(),
755        });
756        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
757        let here_doc2 = Rc::new(HereDoc {
758            delimiter: delimiter2,
759            remove_tabs: true,
760            content: OnceCell::new(),
761        });
762        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
763        let here_doc3 = Rc::new(HereDoc {
764            delimiter: delimiter3,
765            remove_tabs: false,
766            content: OnceCell::new(),
767        });
768        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
769        parser.here_doc_contents().now_or_never().unwrap().unwrap();
770        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
771        assert_eq!(here_doc1.remove_tabs, false);
772        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
773        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
774        assert_eq!(here_doc2.remove_tabs, true);
775        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
776        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
777        assert_eq!(here_doc3.remove_tabs, false);
778        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
779    }
780
781    #[test]
782    fn parser_reading_here_doc_contents_twice() {
783        let delimiter1 = "ONE".parse().unwrap();
784        let delimiter2 = "TWO".parse().unwrap();
785
786        let mut lexer = Lexer::from_memory("1\nONE\n2\nTWO\n", Source::Unknown);
787        #[allow(clippy::mutable_key_type)]
788        let aliases = AliasSet::new();
789        let mut parser = Parser::new(&mut lexer, &aliases);
790        let here_doc1 = Rc::new(HereDoc {
791            delimiter: delimiter1,
792            remove_tabs: false,
793            content: OnceCell::new(),
794        });
795        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
796        parser.here_doc_contents().now_or_never().unwrap().unwrap();
797        let here_doc2 = Rc::new(HereDoc {
798            delimiter: delimiter2,
799            remove_tabs: true,
800            content: OnceCell::new(),
801        });
802        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
803        parser.here_doc_contents().now_or_never().unwrap().unwrap();
804        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
805        assert_eq!(here_doc1.remove_tabs, false);
806        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
807        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
808        assert_eq!(here_doc2.remove_tabs, true);
809        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
810    }
811
812    #[test]
813    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
814    fn parser_here_doc_contents_must_be_called_without_pending_token() {
815        let mut lexer = Lexer::from_memory("X", Source::Unknown);
816        #[allow(clippy::mutable_key_type)]
817        let aliases = AliasSet::new();
818        let mut parser = Parser::new(&mut lexer, &aliases);
819        parser.peek_token().now_or_never().unwrap().unwrap();
820        parser.here_doc_contents().now_or_never().unwrap().unwrap();
821    }
822}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
