yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::syntax::HereDoc;
31use crate::syntax::MaybeLiteral;
32use crate::syntax::Word;
33use std::rc::Rc;
34
35/// Entire result of parsing
36pub type Result<T> = std::result::Result<T, Error>;
37
38/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
39/// parsing after alias substitution
40///
41/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
42///
43/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
44/// meaningful value and is returned from a parsing function that has performed alias substitution
45/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
46/// new source code produced by the substitution so that the syntax is correctly recognized in the
47/// new code.
48///
49/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
50/// recognized as an alias name during parse of a simple command, the simple command parser
51/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
52/// parser function, to recognize the `!` reserved word token as negation.
53///
54/// When a parser function successfully parses something, it returns the result in the `Parsed`
55/// variant. The caller then continues the remaining parse.
56#[derive(Copy, Clone, Debug, Eq, PartialEq)]
57pub enum Rec<T> {
58    /// Result of alias substitution
59    AliasSubstituted,
60    /// Successful parse result
61    Parsed(T),
62}
63
64impl<T> Rec<T> {
65    /// Tests if `self` is `AliasSubstituted`.
66    pub fn is_alias_substituted(&self) -> bool {
67        match self {
68            Rec::AliasSubstituted => true,
69            Rec::Parsed(_) => false,
70        }
71    }
72
73    /// Extracts the result of successful parsing.
74    ///
75    /// # Panics
76    ///
77    /// If `self` is `AliasSubstituted`.
78    pub fn unwrap(self) -> T {
79        match self {
80            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
81            Rec::Parsed(v) => v,
82        }
83    }
84
85    /// Transforms the result value in `self`.
86    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
87    where
88        F: FnOnce(T) -> Result<U>,
89    {
90        match self {
91            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
92            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
93        }
94    }
95}
96
97/// Set of parameters for constructing a [`Parser`]
98///
99/// `Config` is a builder for constructing a parser. A [new](Self::new)
100/// configuration starts with default settings. You can customize them by
101/// calling methods that can be chained. Finally, you can create a parser by
102/// providing the lexer to the [`input`](Self::input) method.
103///
104/// Note that this struct is for configuring a parser. To configure the
105/// [`Lexer`], use [`yash_env::parser::Config`].
106#[derive(Debug)]
107#[must_use = "Config must be used to create a parser"]
108pub struct Config<'a> {
109    /// Collection of aliases the parser applies to substitute command words
110    aliases: &'a dyn crate::alias::Glossary,
111
112    /// Glossary that determines whether a command name is a declaration utility
113    decl_utils: &'a dyn crate::decl_util::Glossary,
114}
115
116impl<'a> Config<'a> {
117    /// Creates a new configuration with default settings.
118    ///
119    /// You can also call [`Parser::config`] to create a new configuration.
120    pub fn new() -> Self {
121        Self {
122            aliases: &crate::alias::EmptyGlossary,
123            decl_utils: &crate::decl_util::PosixGlossary,
124        }
125    }
126
127    /// Sets the glossary of aliases.
128    ///
129    /// The parser uses the glossary to look up aliases and substitute command
130    /// words. The default glossary is [empty](crate::alias::EmptyGlossary).
131    #[inline]
132    pub fn aliases(&mut self, aliases: &'a dyn Glossary) -> &mut Self {
133        self.aliases = aliases;
134        self
135    }
136
137    /// Sets the glossary of declaration utilities.
138    ///
139    /// The parser uses the glossary to determine whether a command name is a
140    /// declaration utility. The default glossary is [`PosixGlossary`], which
141    /// recognizes the declaration utilities defined by POSIX. You can make
142    /// arbitrary command names declaration utilities by providing a custom
143    /// glossary. To meet the POSIX standard, the glossary's
144    /// [`is_declaration_utility`] method must return:
145    ///
146    /// - `Some(true)` for `export` and `readonly`
147    /// - `None` for `command`
148    ///
149    /// For detailed information on declaration utilities, see the
150    /// [`decl_utils`] module.
151    ///
152    /// [`decl_utils`]: crate::decl_util
153    /// [`PosixGlossary`]: crate::decl_util::PosixGlossary
154    /// [`is_declaration_utility`]: crate::decl_util::Glossary::is_declaration_utility
155    #[inline]
156    pub fn declaration_utilities(
157        &mut self,
158        decl_utils: &'a dyn crate::decl_util::Glossary,
159    ) -> &mut Self {
160        self.decl_utils = decl_utils;
161        self
162    }
163
164    /// Creates a parser with the given lexer.
165    pub fn input<'b>(&self, lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
166        Parser {
167            lexer,
168            aliases: self.aliases,
169            decl_utils: self.decl_utils,
170            token: None,
171            unread_here_docs: Vec::new(),
172        }
173    }
174}
175
176impl Default for Config<'_> {
177    fn default() -> Self {
178        Self::new()
179    }
180}
181
182/// The shell syntax parser
183///
184/// A parser manages a set of data used in syntax parsing. It keeps a reference
185/// to a [lexer](Lexer) that provides tokens to parse. It also has some
186/// parameters that can be set by a [configuration](Config) and affect the
187/// parsing process.
188///
189/// The [`new`](Self::new) function directly creates a parser with default
190/// settings. If you want to customize the settings, you can use the
191/// [`config`](Self::config) function to create a configuration and then create a
192/// parser with the configuration.
193///
194/// # Parsing here-documents
195///
196/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
197/// with empty content. The parser creates the `HereDoc` instance when it finds
198/// a here-document operator, but it has not read its content at that time. When
199/// finding a newline token, the parser reads the content and fills it into the
200/// `HereDoc` instance.
201///
202/// Unless you are interested in parsing a specific syntactic construct that is
203/// only part of source code, you will want to use a function that returns a
204/// complete result filled with proper here-document contents if any.
205/// Then the [`command_line`](Self::command_line) function is for you.
206/// See also the [module documentation](super).
207#[derive(Debug)]
208#[must_use = "Parser must be used to parse syntax"]
209pub struct Parser<'a, 'b> {
210    /// Lexer that provides tokens
211    lexer: &'a mut Lexer<'b>,
212
213    /// Collection of aliases the parser applies to substitute command words
214    aliases: &'a dyn crate::alias::Glossary,
215
216    /// Glossary that determines whether a command name is a declaration utility
217    decl_utils: &'a dyn crate::decl_util::Glossary,
218
219    /// Token to parse next
220    ///
221    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
222    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
223    token: Option<Result<Token>>,
224
225    /// Here-documents without contents
226    ///
227    /// The here-document is added to this list when the parser finds a
228    /// here-document operator. After consuming the next newline token, the
229    /// parser reads and fills the contents, then clears this list.
230    unread_here_docs: Vec<Rc<HereDoc>>,
231}
232
233impl<'a, 'b> Parser<'a, 'b> {
234    /// Creates a new configuration with default settings.
235    ///
236    /// This is a synonym for [`Config::new`]. Customize the settings by calling
237    /// methods of the returned configuration and then create a parser by calling
238    /// its [`input`](Config::input) method.
239    #[inline(always)]
240    pub fn config() -> Config<'a> {
241        Config::new()
242    }
243
244    /// Creates a new parser based on the given lexer.
245    ///
246    /// The parser uses the lexer to read tokens. All other settings are default.
247    /// To customize the settings, use the [`config`](Self::config) function.
248    pub fn new(lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
249        Self::config().input(lexer)
250    }
251
252    /// Reads a next token if the current token is `None`.
253    async fn require_token(&mut self) {
254        #[allow(clippy::question_mark)] // TODO https://github.com/rust-lang/rust-clippy/issues/9518
255        if self.token.is_none() {
256            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
257                Err(e)
258            } else {
259                self.lexer.token().await
260            });
261        }
262    }
263
264    /// Returns a reference to the current token.
265    ///
266    /// If the current token is not yet read from the underlying lexer, it is read.
267    pub async fn peek_token(&mut self) -> Result<&Token> {
268        self.require_token().await;
269        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
270    }
271
272    /// Consumes the current token without performing alias substitution.
273    ///
274    /// If the current token is not yet read from the underlying lexer, it is read.
275    ///
276    /// This function does not perform alias substitution and therefore should be
277    /// used only in context where no alias substitution is expected. Otherwise,
278    /// you should use [`take_token_manual`](Self::take_token_manual) or
279    /// [`take_token_auto`](Self::take_token_auto) instead.
280    pub async fn take_token_raw(&mut self) -> Result<Token> {
281        self.require_token().await;
282        self.token.take().unwrap()
283    }
284
285    /// Performs alias substitution on a token that has just been
286    /// [taken](Self::take_token_raw).
287    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
288        // TODO Only POSIXly-valid alias name should be recognized in POSIXly-correct mode.
289        if !self.aliases.is_empty() {
290            if let Token(_) = token.id {
291                if let Some(name) = token.word.to_string_if_literal() {
292                    if !token.word.location.code.source.is_alias_for(&name) {
293                        if let Some(alias) = self.aliases.look_up(&name) {
294                            if is_command_name
295                                || alias.global
296                                || self.lexer.is_after_blank_ending_alias(token.index)
297                            {
298                                self.lexer.substitute_alias(token.index, &alias);
299                                return Rec::AliasSubstituted;
300                            }
301                        }
302                    }
303                }
304            }
305        }
306
307        Rec::Parsed(token)
308    }
309
310    /// Consumes the current token after performing applicable alias substitution.
311    ///
312    /// If the current token is not yet read from the underlying lexer, it is read.
313    ///
314    /// This function checks if the token is the name of an alias. If it is,
315    /// alias substitution is performed on the token and the result is
316    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
317    ///
318    /// Alias substitution is performed only if at least one of the following is
319    /// true:
320    ///
321    /// - The token is the first command word in a simple command, that is, it is
322    ///   the word for the command name. (This condition should be specified by the
323    ///   `is_command_name` parameter.)
324    /// - The token comes just after the replacement string of another alias
325    ///   substitution that ends with a blank character.
326    /// - The token names a global alias.
327    ///
328    /// However, alias substitution should _not_ be performed on a reserved word
329    /// in any case. It is your responsibility to check the token type and not to
330    /// call this function on a reserved word. That is why this function is named
331    /// `manual`. To consume a reserved word without performing alias
332    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
333    /// [`take_token_auto`](Self::take_token_auto).
334    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
335        let token = self.take_token_raw().await?;
336        Ok(self.substitute_alias(token, is_command_name))
337    }
338
339    /// Consumes the current token after performing applicable alias substitution.
340    ///
341    /// This function performs alias substitution unless the result is one of the
342    /// reserved words specified in the argument.
343    ///
344    /// Alias substitution is performed repeatedly until a non-alias token is
345    /// found. That is why this function is named `auto`. This function should be
346    /// used only in contexts where no backtrack is needed after alias
347    /// substitution. If you need to backtrack or want to know whether alias
348    /// substitution was performed or not, you should use
349    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
350    /// alias substitution at most once and returns `Rec`.
351    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
352        loop {
353            let token = self.take_token_raw().await?;
354            if let Token(Some(keyword)) = token.id {
355                if keywords.contains(&keyword) {
356                    return Ok(token);
357                }
358            }
359            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
360                return Ok(token);
361            }
362        }
363    }
364
365    /// Tests if there is a blank before the next token.
366    ///
367    /// This function can be called to tell whether the previous and next tokens
368    /// are separated by a blank or they are adjacent.
369    ///
370    /// This function must be called after the previous token has been taken (by
371    /// one of [`take_token_raw`](Self::take_token_raw),
372    /// [`take_token_manual`](Self::take_token_manual) and
373    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
374    /// [peeked](Self::peek_token). Otherwise, this function would panic.
375    ///
376    /// # Panics
377    ///
378    /// If the previous token has not been taken or the next token has been
379    /// peeked.
380    pub async fn has_blank(&mut self) -> Result<bool> {
381        assert!(self.token.is_none(), "There should be no pending token");
382        let c = self.lexer.peek_char().await?;
383        Ok(c.is_some_and(is_blank))
384    }
385
386    /// Remembers the given partial here-document for later parsing of its content.
387    ///
388    /// The remembered here-document's content will be parsed when
389    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
390    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
391        self.unread_here_docs.push(here_doc)
392    }
393
394    /// Reads here-document contents that matches the remembered list of
395    /// here-document operators.
396    ///
397    /// This function reads here-document contents corresponding to
398    /// here-document operators that have been saved with
399    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
400    /// The results are inserted to the `content` field of the `HereDoc`
401    /// instances.
402    ///
403    /// This function must be called just after a newline token has been taken
404    /// (either [manual](Self::take_token_manual) or
405    /// [auto](Self::take_token_auto)). If there is a pending token that has been
406    /// peeked but not yet taken, this function will panic!
407    pub async fn here_doc_contents(&mut self) -> Result<()> {
408        assert!(
409            self.token.is_none(),
410            "No token must be peeked before reading here-doc contents"
411        );
412
413        for here_doc in self.unread_here_docs.drain(..) {
414            self.lexer.here_doc_content(&here_doc).await?;
415        }
416
417        Ok(())
418    }
419
420    /// Ensures that there is no pending partial here-document.
421    ///
422    /// If there is any, this function returns a `MissingHereDocContent` error.
423    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
424        match self.unread_here_docs.first() {
425            None => Ok(()),
426            Some(here_doc) => Err(Error {
427                cause: SyntaxError::MissingHereDocContent.into(),
428                location: here_doc.delimiter.location.clone(),
429            }),
430        }
431    }
432
433    /// Determines whether a word names a declaration utility.
434    ///
435    /// See [`decl_utils`](crate::decl_util) for more information.
436    pub(super) fn word_names_declaration_utility(&self, word: &Word) -> Option<bool> {
437        if let Some(name) = word.to_string_if_literal() {
438            self.decl_utils.is_declaration_utility(&name)
439        } else {
440            Some(false)
441        }
442    }
443}
444
445#[allow(clippy::bool_assert_comparison)]
446#[cfg(test)]
447mod tests {
448    use super::*;
449    use crate::alias::AliasSet;
450    use crate::alias::HashEntry;
451    use crate::source::Location;
452    use assert_matches::assert_matches;
453    use futures_util::FutureExt;
454    use std::cell::OnceCell;
455
456    #[test]
457    fn parser_take_token_manual_successful_substitution() {
458        let mut lexer = Lexer::with_code("X");
459        #[allow(clippy::mutable_key_type)]
460        let mut aliases = AliasSet::new();
461        aliases.insert(HashEntry::new(
462            "X".to_string(),
463            "x".to_string(),
464            false,
465            Location::dummy("?"),
466        ));
467        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
468
469        let result = parser.take_token_manual(true).now_or_never().unwrap();
470        assert_matches!(result, Ok(Rec::AliasSubstituted));
471
472        let result = parser.take_token_manual(true).now_or_never().unwrap();
473        let token = result.unwrap().unwrap();
474        assert_eq!(token.to_string(), "x");
475    }
476
477    #[test]
478    fn parser_take_token_manual_not_command_name() {
479        let mut lexer = Lexer::with_code("X");
480        #[allow(clippy::mutable_key_type)]
481        let mut aliases = AliasSet::new();
482        aliases.insert(HashEntry::new(
483            "X".to_string(),
484            "x".to_string(),
485            false,
486            Location::dummy("?"),
487        ));
488        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
489
490        let result = parser.take_token_manual(false).now_or_never().unwrap();
491        let token = result.unwrap().unwrap();
492        assert_eq!(token.to_string(), "X");
493    }
494
495    #[test]
496    fn parser_take_token_manual_not_literal() {
497        let mut lexer = Lexer::with_code(r"\X");
498        #[allow(clippy::mutable_key_type)]
499        let mut aliases = AliasSet::new();
500        aliases.insert(HashEntry::new(
501            "X".to_string(),
502            "x".to_string(),
503            false,
504            Location::dummy("?"),
505        ));
506        aliases.insert(HashEntry::new(
507            r"\X".to_string(),
508            "quoted".to_string(),
509            false,
510            Location::dummy("?"),
511        ));
512        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
513
514        let result = parser.take_token_manual(true).now_or_never().unwrap();
515        let token = result.unwrap().unwrap();
516        assert_eq!(token.to_string(), r"\X");
517    }
518
519    #[test]
520    fn parser_take_token_manual_operator() {
521        let mut lexer = Lexer::with_code(";");
522        #[allow(clippy::mutable_key_type)]
523        let mut aliases = AliasSet::new();
524        aliases.insert(HashEntry::new(
525            ";".to_string(),
526            "x".to_string(),
527            false,
528            Location::dummy("?"),
529        ));
530        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
531
532        let result = parser.take_token_manual(true).now_or_never().unwrap();
533        let token = result.unwrap().unwrap();
534        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
535        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
536    }
537
538    #[test]
539    fn parser_take_token_manual_no_match() {
540        let mut lexer = Lexer::with_code("X");
541        let mut parser = Parser::new(&mut lexer);
542
543        let result = parser.take_token_manual(true).now_or_never().unwrap();
544        let token = result.unwrap().unwrap();
545        assert_eq!(token.to_string(), "X");
546    }
547
548    #[test]
549    fn parser_take_token_manual_recursive_substitution() {
550        let mut lexer = Lexer::with_code("X");
551        #[allow(clippy::mutable_key_type)]
552        let mut aliases = AliasSet::new();
553        aliases.insert(HashEntry::new(
554            "X".to_string(),
555            "Y x".to_string(),
556            false,
557            Location::dummy("?"),
558        ));
559        aliases.insert(HashEntry::new(
560            "Y".to_string(),
561            "X y".to_string(),
562            false,
563            Location::dummy("?"),
564        ));
565        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
566
567        let result = parser.take_token_manual(true).now_or_never().unwrap();
568        assert_matches!(result, Ok(Rec::AliasSubstituted));
569
570        let result = parser.take_token_manual(true).now_or_never().unwrap();
571        assert_matches!(result, Ok(Rec::AliasSubstituted));
572
573        let result = parser.take_token_manual(true).now_or_never().unwrap();
574        let token = result.unwrap().unwrap();
575        assert_eq!(token.to_string(), "X");
576
577        let result = parser.take_token_manual(true).now_or_never().unwrap();
578        let token = result.unwrap().unwrap();
579        assert_eq!(token.to_string(), "y");
580
581        let rec = parser.take_token_manual(true).now_or_never().unwrap();
582        let token = rec.unwrap().unwrap();
583        assert_eq!(token.to_string(), "x");
584    }
585
586    #[test]
587    fn parser_take_token_manual_after_blank_ending_substitution() {
588        let mut lexer = Lexer::with_code("X\tY");
589        #[allow(clippy::mutable_key_type)]
590        let mut aliases = AliasSet::new();
591        aliases.insert(HashEntry::new(
592            "X".to_string(),
593            " X ".to_string(),
594            false,
595            Location::dummy("?"),
596        ));
597        aliases.insert(HashEntry::new(
598            "Y".to_string(),
599            "y".to_string(),
600            false,
601            Location::dummy("?"),
602        ));
603        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
604
605        let result = parser.take_token_manual(true).now_or_never().unwrap();
606        assert_matches!(result, Ok(Rec::AliasSubstituted));
607
608        let result = parser.take_token_manual(true).now_or_never().unwrap();
609        let token = result.unwrap().unwrap();
610        assert_eq!(token.to_string(), "X");
611
612        let result = parser.take_token_manual(false).now_or_never().unwrap();
613        assert_matches!(result, Ok(Rec::AliasSubstituted));
614
615        let result = parser.take_token_manual(false).now_or_never().unwrap();
616        let token = result.unwrap().unwrap();
617        assert_eq!(token.to_string(), "y");
618    }
619
620    #[test]
621    fn parser_take_token_manual_not_after_blank_ending_substitution() {
622        let mut lexer = Lexer::with_code("X\tY");
623        #[allow(clippy::mutable_key_type)]
624        let mut aliases = AliasSet::new();
625        aliases.insert(HashEntry::new(
626            "X".to_string(),
627            " X".to_string(),
628            false,
629            Location::dummy("?"),
630        ));
631        aliases.insert(HashEntry::new(
632            "Y".to_string(),
633            "y".to_string(),
634            false,
635            Location::dummy("?"),
636        ));
637        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
638
639        let result = parser.take_token_manual(true).now_or_never().unwrap();
640        assert_matches!(result, Ok(Rec::AliasSubstituted));
641
642        let result = parser.take_token_manual(true).now_or_never().unwrap();
643        let token = result.unwrap().unwrap();
644        assert_eq!(token.to_string(), "X");
645
646        let result = parser.take_token_manual(false).now_or_never().unwrap();
647        let token = result.unwrap().unwrap();
648        assert_eq!(token.to_string(), "Y");
649    }
650
651    #[test]
652    fn parser_take_token_manual_global() {
653        let mut lexer = Lexer::with_code("X");
654        #[allow(clippy::mutable_key_type)]
655        let mut aliases = AliasSet::new();
656        aliases.insert(HashEntry::new(
657            "X".to_string(),
658            "x".to_string(),
659            true,
660            Location::dummy("?"),
661        ));
662        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
663
664        let result = parser.take_token_manual(false).now_or_never().unwrap();
665        assert_matches!(result, Ok(Rec::AliasSubstituted));
666
667        let result = parser.take_token_manual(false).now_or_never().unwrap();
668        let token = result.unwrap().unwrap();
669        assert_eq!(token.to_string(), "x");
670    }
671
672    #[test]
673    fn parser_take_token_auto_non_keyword() {
674        let mut lexer = Lexer::with_code("X");
675        #[allow(clippy::mutable_key_type)]
676        let mut aliases = AliasSet::new();
677        aliases.insert(HashEntry::new(
678            "X".to_string(),
679            "x".to_string(),
680            true,
681            Location::dummy("?"),
682        ));
683        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
684
685        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
686        assert_eq!(token.to_string(), "x");
687    }
688
689    #[test]
690    fn parser_take_token_auto_keyword_matched() {
691        let mut lexer = Lexer::with_code("if");
692        #[allow(clippy::mutable_key_type)]
693        let mut aliases = AliasSet::new();
694        aliases.insert(HashEntry::new(
695            "if".to_string(),
696            "x".to_string(),
697            true,
698            Location::dummy("?"),
699        ));
700        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
701
702        let token = parser
703            .take_token_auto(&[Keyword::If])
704            .now_or_never()
705            .unwrap()
706            .unwrap();
707        assert_eq!(token.to_string(), "if");
708    }
709
710    #[test]
711    fn parser_take_token_auto_keyword_unmatched() {
712        let mut lexer = Lexer::with_code("if");
713        #[allow(clippy::mutable_key_type)]
714        let mut aliases = AliasSet::new();
715        aliases.insert(HashEntry::new(
716            "if".to_string(),
717            "x".to_string(),
718            true,
719            Location::dummy("?"),
720        ));
721        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
722
723        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
724        assert_eq!(token.to_string(), "x");
725    }
726
727    #[test]
728    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
729        let mut lexer = Lexer::with_code("X");
730        #[allow(clippy::mutable_key_type)]
731        let mut aliases = AliasSet::new();
732        aliases.insert(HashEntry::new(
733            "X".to_string(),
734            "if".to_string(),
735            true,
736            Location::dummy("?"),
737        ));
738        aliases.insert(HashEntry::new(
739            "if".to_string(),
740            "x".to_string(),
741            true,
742            Location::dummy("?"),
743        ));
744        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
745
746        let token = parser
747            .take_token_auto(&[Keyword::If])
748            .now_or_never()
749            .unwrap()
750            .unwrap();
751        assert_eq!(token.to_string(), "if");
752    }
753
754    #[test]
755    fn parser_has_blank_true() {
756        let mut lexer = Lexer::with_code(" ");
757        let mut parser = Parser::new(&mut lexer);
758        let result = parser.has_blank().now_or_never().unwrap();
759        assert_eq!(result, Ok(true));
760    }
761
762    #[test]
763    fn parser_has_blank_false() {
764        let mut lexer = Lexer::with_code("(");
765        let mut parser = Parser::new(&mut lexer);
766        let result = parser.has_blank().now_or_never().unwrap();
767        assert_eq!(result, Ok(false));
768    }
769
770    #[test]
771    fn parser_has_blank_eof() {
772        let mut lexer = Lexer::with_code("");
773        let mut parser = Parser::new(&mut lexer);
774        let result = parser.has_blank().now_or_never().unwrap();
775        assert_eq!(result, Ok(false));
776    }
777
778    #[test]
779    fn parser_has_blank_true_with_line_continuations() {
780        let mut lexer = Lexer::with_code("\\\n\\\n ");
781        let mut parser = Parser::new(&mut lexer);
782        let result = parser.has_blank().now_or_never().unwrap();
783        assert_eq!(result, Ok(true));
784    }
785
786    #[test]
787    fn parser_has_blank_false_with_line_continuations() {
788        let mut lexer = Lexer::with_code("\\\n\\\n\\\n(");
789        let mut parser = Parser::new(&mut lexer);
790        let result = parser.has_blank().now_or_never().unwrap();
791        assert_eq!(result, Ok(false));
792    }
793
794    #[test]
795    #[should_panic(expected = "There should be no pending token")]
796    fn parser_has_blank_with_pending_token() {
797        let mut lexer = Lexer::with_code("foo");
798        let mut parser = Parser::new(&mut lexer);
799        parser.peek_token().now_or_never().unwrap().unwrap();
800        let _ = parser.has_blank().now_or_never().unwrap();
801    }
802
803    #[test]
804    fn parser_reading_no_here_doc_contents() {
805        let mut lexer = Lexer::with_code("X");
806        let mut parser = Parser::new(&mut lexer);
807        parser.here_doc_contents().now_or_never().unwrap().unwrap();
808
809        let location = lexer.location().now_or_never().unwrap().unwrap();
810        assert_eq!(location.code.start_line_number.get(), 1);
811        assert_eq!(location.range, 0..1);
812    }
813
814    #[test]
815    fn parser_reading_one_here_doc_content() {
816        let delimiter = "END".parse().unwrap();
817
818        let mut lexer = Lexer::with_code("END\nX");
819        let mut parser = Parser::new(&mut lexer);
820        let remove_tabs = false;
821        let here_doc = Rc::new(HereDoc {
822            delimiter,
823            remove_tabs,
824            content: OnceCell::new(),
825        });
826        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
827        parser.here_doc_contents().now_or_never().unwrap().unwrap();
828        assert_eq!(here_doc.delimiter.to_string(), "END");
829        assert_eq!(here_doc.remove_tabs, remove_tabs);
830        assert_eq!(here_doc.content.get().unwrap().0, []);
831
832        let location = lexer.location().now_or_never().unwrap().unwrap();
833        assert_eq!(location.code.start_line_number.get(), 1);
834        assert_eq!(location.range, 4..5);
835    }
836
837    #[test]
838    fn parser_reading_many_here_doc_contents() {
839        let delimiter1 = "ONE".parse().unwrap();
840        let delimiter2 = "TWO".parse().unwrap();
841        let delimiter3 = "THREE".parse().unwrap();
842
843        let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX");
844        let mut parser = Parser::new(&mut lexer);
845        let here_doc1 = Rc::new(HereDoc {
846            delimiter: delimiter1,
847            remove_tabs: false,
848            content: OnceCell::new(),
849        });
850        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
851        let here_doc2 = Rc::new(HereDoc {
852            delimiter: delimiter2,
853            remove_tabs: true,
854            content: OnceCell::new(),
855        });
856        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
857        let here_doc3 = Rc::new(HereDoc {
858            delimiter: delimiter3,
859            remove_tabs: false,
860            content: OnceCell::new(),
861        });
862        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
863        parser.here_doc_contents().now_or_never().unwrap().unwrap();
864        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
865        assert_eq!(here_doc1.remove_tabs, false);
866        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
867        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
868        assert_eq!(here_doc2.remove_tabs, true);
869        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
870        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
871        assert_eq!(here_doc3.remove_tabs, false);
872        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
873    }
874
875    #[test]
876    fn parser_reading_here_doc_contents_twice() {
877        let delimiter1 = "ONE".parse().unwrap();
878        let delimiter2 = "TWO".parse().unwrap();
879
880        let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n");
881        let mut parser = Parser::new(&mut lexer);
882        let here_doc1 = Rc::new(HereDoc {
883            delimiter: delimiter1,
884            remove_tabs: false,
885            content: OnceCell::new(),
886        });
887        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
888        parser.here_doc_contents().now_or_never().unwrap().unwrap();
889        let here_doc2 = Rc::new(HereDoc {
890            delimiter: delimiter2,
891            remove_tabs: true,
892            content: OnceCell::new(),
893        });
894        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
895        parser.here_doc_contents().now_or_never().unwrap().unwrap();
896        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
897        assert_eq!(here_doc1.remove_tabs, false);
898        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
899        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
900        assert_eq!(here_doc2.remove_tabs, true);
901        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
902    }
903
904    #[test]
905    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
906    fn parser_here_doc_contents_must_be_called_without_pending_token() {
907        let mut lexer = Lexer::with_code("X");
908        let mut parser = Parser::new(&mut lexer);
909        parser.peek_token().now_or_never().unwrap().unwrap();
910        parser.here_doc_contents().now_or_never().unwrap().unwrap();
911    }
912}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
