yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::source::Location;
31use crate::syntax::HereDoc;
32use crate::syntax::MaybeLiteral as _;
33use crate::syntax::Word;
34use std::rc::Rc;
35
36/// Entire result of parsing
37pub type Result<T> = std::result::Result<T, Error>;
38
39/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
40/// parsing after alias substitution
41///
42/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
43///
44/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
45/// meaningful value and is returned from a parsing function that has performed alias substitution
46/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
47/// new source code produced by the substitution so that the syntax is correctly recognized in the
48/// new code.
49///
50/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
51/// recognized as an alias name during parse of a simple command, the simple command parser
52/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
53/// parser function, to recognize the `!` reserved word token as negation.
54///
55/// When a parser function successfully parses something, it returns the result in the `Parsed`
56/// variant. The caller then continues the remaining parse.
57#[derive(Copy, Clone, Debug, Eq, PartialEq)]
58pub enum Rec<T> {
59    /// Result of alias substitution
60    AliasSubstituted,
61    /// Successful parse result
62    Parsed(T),
63}
64
65impl<T> Rec<T> {
66    /// Tests if `self` is `AliasSubstituted`.
67    pub fn is_alias_substituted(&self) -> bool {
68        match self {
69            Rec::AliasSubstituted => true,
70            Rec::Parsed(_) => false,
71        }
72    }
73
74    /// Extracts the result of successful parsing.
75    ///
76    /// # Panics
77    ///
78    /// If `self` is `AliasSubstituted`.
79    pub fn unwrap(self) -> T {
80        match self {
81            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
82            Rec::Parsed(v) => v,
83        }
84    }
85
86    /// Transforms the result value in `self`.
87    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
88    where
89        F: FnOnce(T) -> Result<U>,
90    {
91        match self {
92            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
93            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
94        }
95    }
96}
97
98/// Set of parameters for constructing a [`Parser`]
99///
100/// `Config` is a builder for constructing a parser. A [new](Self::new)
101/// configuration starts with default settings. You can customize them by
102/// calling methods that can be chained. Finally, you can create a parser by
103/// providing the lexer to the [`input`](Self::input) method.
104///
105/// Note that this struct is for configuring a parser. To configure the
106/// [`Lexer`], use [`yash_env::parser::Config`].
107#[derive(Debug)]
108#[must_use = "Config must be used to create a parser"]
109pub struct Config<'a> {
110    /// Collection of aliases the parser applies to substitute command words
111    aliases: &'a dyn crate::alias::Glossary,
112
113    /// Glossary that determines whether a command name is a declaration utility
114    decl_utils: &'a dyn crate::decl_util::Glossary,
115}
116
117impl<'a> Config<'a> {
118    /// Creates a new configuration with default settings.
119    ///
120    /// You can also call [`Parser::config`] to create a new configuration.
121    pub fn new() -> Self {
122        Self {
123            aliases: &crate::alias::EmptyGlossary,
124            decl_utils: &crate::decl_util::PosixGlossary,
125        }
126    }
127
128    /// Sets the glossary of aliases.
129    ///
130    /// The parser uses the glossary to look up aliases and substitute command
131    /// words. The default glossary is [empty](crate::alias::EmptyGlossary).
132    #[inline]
133    pub fn aliases(&mut self, aliases: &'a dyn Glossary) -> &mut Self {
134        self.aliases = aliases;
135        self
136    }
137
138    /// Sets the glossary of declaration utilities.
139    ///
140    /// The parser uses the glossary to determine whether a command name is a
141    /// declaration utility. The default glossary is [`PosixGlossary`], which
142    /// recognizes the declaration utilities defined by POSIX. You can make
143    /// arbitrary command names declaration utilities by providing a custom
144    /// glossary. To meet the POSIX standard, the glossary's
145    /// [`is_declaration_utility`] method must return:
146    ///
147    /// - `Some(true)` for `export` and `readonly`
148    /// - `None` for `command`
149    ///
150    /// For detailed information on declaration utilities, see the
151    /// [`decl_utils`] module.
152    ///
153    /// [`decl_utils`]: crate::decl_util
154    /// [`PosixGlossary`]: crate::decl_util::PosixGlossary
155    /// [`is_declaration_utility`]: crate::decl_util::Glossary::is_declaration_utility
156    #[inline]
157    pub fn declaration_utilities(
158        &mut self,
159        decl_utils: &'a dyn crate::decl_util::Glossary,
160    ) -> &mut Self {
161        self.decl_utils = decl_utils;
162        self
163    }
164
165    /// Creates a parser with the given lexer.
166    pub fn input<'b>(&self, lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
167        Parser {
168            lexer,
169            aliases: self.aliases,
170            decl_utils: self.decl_utils,
171            token: None,
172            unread_here_docs: Vec::new(),
173        }
174    }
175}
176
177impl Default for Config<'_> {
178    fn default() -> Self {
179        Self::new()
180    }
181}
182
183/// The shell syntax parser
184///
185/// A parser manages a set of data used in syntax parsing. It keeps a reference
186/// to a [lexer](Lexer) that provides tokens to parse. It also has some
187/// parameters that can be set by a [configuration](Config) and affect the
188/// parsing process.
189///
190/// The [`new`](Self::new) function directly creates a parser with default
191/// settings. If you want to customize the settings, you can use the
192/// [`config`](Self::config) function to create a configuration and then create a
193/// parser with the configuration.
194///
195/// # Parsing here-documents
196///
197/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
198/// with empty content. The parser creates the `HereDoc` instance when it finds
199/// a here-document operator, but it has not read its content at that time. When
200/// finding a newline token, the parser reads the content and fills it into the
201/// `HereDoc` instance.
202///
203/// Unless you are interested in parsing a specific syntactic construct that is
204/// only part of source code, you will want to use a function that returns a
205/// complete result filled with proper here-document contents if any.
206/// Then the [`command_line`](Self::command_line) function is for you.
207/// See also the [module documentation](super).
208#[derive(Debug)]
209#[must_use = "Parser must be used to parse syntax"]
210pub struct Parser<'a, 'b> {
211    /// Lexer that provides tokens
212    lexer: &'a mut Lexer<'b>,
213
214    /// Collection of aliases the parser applies to substitute command words
215    aliases: &'a dyn crate::alias::Glossary,
216
217    /// Glossary that determines whether a command name is a declaration utility
218    decl_utils: &'a dyn crate::decl_util::Glossary,
219
220    /// Token to parse next
221    ///
222    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
223    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
224    token: Option<Result<Token>>,
225
226    /// Here-documents without contents
227    ///
228    /// The here-document is added to this list when the parser finds a
229    /// here-document operator. After consuming the next newline token, the
230    /// parser reads and fills the contents, then clears this list.
231    unread_here_docs: Vec<Rc<HereDoc>>,
232}
233
234impl<'a, 'b> Parser<'a, 'b> {
235    /// Creates a new configuration with default settings.
236    ///
237    /// This is a synonym for [`Config::new`]. Customize the settings by calling
238    /// methods of the returned configuration and then create a parser by calling
239    /// its [`input`](Config::input) method.
240    #[inline(always)]
241    pub fn config() -> Config<'a> {
242        Config::new()
243    }
244
245    /// Creates a new parser based on the given lexer.
246    ///
247    /// The parser uses the lexer to read tokens. All other settings are default.
248    /// To customize the settings, use the [`config`](Self::config) function.
249    pub fn new(lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
250        Self::config().input(lexer)
251    }
252
253    /// Reads a next token if the current token is `None`.
254    async fn require_token(&mut self) {
255        #[allow(clippy::question_mark, reason = "false positive")]
256        // TODO https://github.com/rust-lang/rust-clippy/issues/9518
257        if self.token.is_none() {
258            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
259                Err(e)
260            } else {
261                self.lexer.token().await
262            });
263        }
264    }
265
266    /// Returns the parsing mode of the underlying lexer.
267    ///
268    /// The parser consults the mode to decide which syntax to accept. See
269    /// [`Lexer::mode`](crate::parser::lex::Lexer::mode).
270    pub(super) fn mode(&self) -> yash_env::parser::Mode {
271        self.lexer.mode()
272    }
273
274    /// Returns a reference to the current token.
275    ///
276    /// If the current token is not yet read from the underlying lexer, it is read.
277    pub async fn peek_token(&mut self) -> Result<&Token> {
278        self.require_token().await;
279        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
280    }
281
282    /// Consumes the current token without performing alias substitution.
283    ///
284    /// If the current token is not yet read from the underlying lexer, it is read.
285    ///
286    /// This function does not perform alias substitution and therefore should be
287    /// used only in context where no alias substitution is expected. Otherwise,
288    /// you should use [`take_token_manual`](Self::take_token_manual) or
289    /// [`take_token_auto`](Self::take_token_auto) instead.
290    pub async fn take_token_raw(&mut self) -> Result<Token> {
291        self.require_token().await;
292        self.token.take().unwrap()
293    }
294
295    /// Performs alias substitution on a token that has just been
296    /// [taken](Self::take_token_raw).
297    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
298        if !self.aliases.is_empty()
299            && let Token(_) = token.id
300            && let Some(name) = token.word.to_string_if_literal()
301            && !token.word.location.code.source.is_alias_for(&name)
302            && let Some(alias) = self.aliases.look_up(&name)
303            && (is_command_name
304                || alias.global
305                || self.lexer.is_after_blank_ending_alias(token.index))
306        {
307            self.lexer.substitute_alias(token.index, &alias);
308            return Rec::AliasSubstituted;
309        }
310
311        Rec::Parsed(token)
312    }
313
314    /// Consumes the current token after performing applicable alias substitution.
315    ///
316    /// If the current token is not yet read from the underlying lexer, it is read.
317    ///
318    /// This function checks if the token is the name of an alias. If it is,
319    /// alias substitution is performed on the token and the result is
320    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
321    ///
322    /// Alias substitution is performed only if at least one of the following is
323    /// true:
324    ///
325    /// - The token is the first command word in a simple command, that is, it is
326    ///   the word for the command name. (This condition should be specified by the
327    ///   `is_command_name` parameter.)
328    /// - The token comes just after the replacement string of another alias
329    ///   substitution that ends with a blank character.
330    /// - The token names a global alias.
331    ///
332    /// However, alias substitution should _not_ be performed on a reserved word
333    /// in any case. It is your responsibility to check the token type and not to
334    /// call this function on a reserved word. That is why this function is named
335    /// `manual`. To consume a reserved word without performing alias
336    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
337    /// [`take_token_auto`](Self::take_token_auto).
338    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
339        let token = self.take_token_raw().await?;
340        Ok(self.substitute_alias(token, is_command_name))
341    }
342
343    /// Consumes the current token after performing applicable alias substitution.
344    ///
345    /// This function performs alias substitution unless the result is one of the
346    /// reserved words specified in the argument.
347    ///
348    /// Alias substitution is performed repeatedly until a non-alias token is
349    /// found. That is why this function is named `auto`. This function should be
350    /// used only in contexts where no backtrack is needed after alias
351    /// substitution. If you need to backtrack or want to know whether alias
352    /// substitution was performed or not, you should use
353    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
354    /// alias substitution at most once and returns `Rec`.
355    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
356        loop {
357            let token = self.take_token_raw().await?;
358            if let Token(Some(keyword)) = token.id
359                && keywords.contains(&keyword)
360            {
361                return Ok(token);
362            }
363            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
364                return Ok(token);
365            }
366        }
367    }
368
369    /// Tests if there is a blank before the next token.
370    ///
371    /// This function can be called to tell whether the previous and next tokens
372    /// are separated by a blank or they are adjacent.
373    ///
374    /// This function must be called after the previous token has been taken (by
375    /// one of [`take_token_raw`](Self::take_token_raw),
376    /// [`take_token_manual`](Self::take_token_manual) and
377    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
378    /// [peeked](Self::peek_token). Otherwise, this function would panic.
379    ///
380    /// # Panics
381    ///
382    /// If the previous token has not been taken or the next token has been
383    /// peeked.
384    pub async fn has_blank(&mut self) -> Result<bool> {
385        assert!(self.token.is_none(), "There should be no pending token");
386        let c = self.lexer.peek_char().await?;
387        Ok(c.is_some_and(is_blank))
388    }
389
390    /// Returns the location of the current position.
391    ///
392    /// This function must be called after the previous token has been taken (by
393    /// one of [`take_token_raw`](Self::take_token_raw),
394    /// [`take_token_manual`](Self::take_token_manual) and
395    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
396    /// [peeked](Self::peek_token). Otherwise, this function would panic.
397    ///
398    /// # Panics
399    ///
400    /// If the previous token has not been taken or the next token has been
401    /// peeked.
402    pub(super) async fn location(&mut self) -> Result<&Location> {
403        assert!(self.token.is_none(), "There should be no pending token");
404        self.lexer.location().await
405    }
406
407    /// Remembers the given partial here-document for later parsing of its content.
408    ///
409    /// The remembered here-document's content will be parsed when
410    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
411    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
412        self.unread_here_docs.push(here_doc)
413    }
414
415    /// Reads here-document contents that matches the remembered list of
416    /// here-document operators.
417    ///
418    /// This function reads here-document contents corresponding to
419    /// here-document operators that have been saved with
420    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
421    /// The results are inserted to the `content` field of the `HereDoc`
422    /// instances.
423    ///
424    /// This function must be called just after a newline token has been taken
425    /// (either [manual](Self::take_token_manual) or
426    /// [auto](Self::take_token_auto)). If there is a pending token that has been
427    /// peeked but not yet taken, this function will panic!
428    pub async fn here_doc_contents(&mut self) -> Result<()> {
429        assert!(
430            self.token.is_none(),
431            "No token must be peeked before reading here-doc contents"
432        );
433
434        for here_doc in self.unread_here_docs.drain(..) {
435            self.lexer.here_doc_content(&here_doc).await?;
436        }
437
438        Ok(())
439    }
440
441    /// Ensures that there is no pending partial here-document.
442    ///
443    /// If there is any, this function returns a `MissingHereDocContent` error.
444    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
445        match self.unread_here_docs.first() {
446            None => Ok(()),
447            Some(here_doc) => Err(Error {
448                cause: SyntaxError::MissingHereDocContent.into(),
449                location: here_doc.delimiter.location.clone(),
450            }),
451        }
452    }
453
454    /// Determines whether a word names a declaration utility.
455    ///
456    /// See [`decl_utils`](crate::decl_util) for more information.
457    pub(super) fn word_names_declaration_utility(&self, word: &Word) -> Option<bool> {
458        if let Some(name) = word.to_string_if_literal() {
459            self.decl_utils.is_declaration_utility(&name)
460        } else {
461            Some(false)
462        }
463    }
464}
465
466#[allow(
467    clippy::bool_assert_comparison,
468    reason = "to make the expected values clearer"
469)]
470#[cfg(test)]
471mod tests {
472    use super::*;
473    use crate::alias::AliasSet;
474    use crate::alias::HashEntry;
475    use futures_util::FutureExt as _;
476    use std::assert_matches;
477    use std::cell::OnceCell;
478
479    #[test]
480    fn parser_take_token_manual_successful_substitution() {
481        let mut lexer = Lexer::with_code("X");
482        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
483        let mut aliases = AliasSet::new();
484        aliases.insert(HashEntry::new(
485            "X".to_string(),
486            "x".to_string(),
487            false,
488            Location::dummy("?"),
489        ));
490        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
491
492        let result = parser.take_token_manual(true).now_or_never().unwrap();
493        assert_matches!(result, Ok(Rec::AliasSubstituted));
494
495        let result = parser.take_token_manual(true).now_or_never().unwrap();
496        let token = result.unwrap().unwrap();
497        assert_eq!(token.to_string(), "x");
498    }
499
500    #[test]
501    fn parser_take_token_manual_not_command_name() {
502        let mut lexer = Lexer::with_code("X");
503        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
504        let mut aliases = AliasSet::new();
505        aliases.insert(HashEntry::new(
506            "X".to_string(),
507            "x".to_string(),
508            false,
509            Location::dummy("?"),
510        ));
511        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
512
513        let result = parser.take_token_manual(false).now_or_never().unwrap();
514        let token = result.unwrap().unwrap();
515        assert_eq!(token.to_string(), "X");
516    }
517
518    #[test]
519    fn parser_take_token_manual_not_literal() {
520        let mut lexer = Lexer::with_code(r"\X");
521        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
522        let mut aliases = AliasSet::new();
523        aliases.insert(HashEntry::new(
524            "X".to_string(),
525            "x".to_string(),
526            false,
527            Location::dummy("?"),
528        ));
529        aliases.insert(HashEntry::new(
530            r"\X".to_string(),
531            "quoted".to_string(),
532            false,
533            Location::dummy("?"),
534        ));
535        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
536
537        let result = parser.take_token_manual(true).now_or_never().unwrap();
538        let token = result.unwrap().unwrap();
539        assert_eq!(token.to_string(), r"\X");
540    }
541
542    #[test]
543    fn parser_take_token_manual_operator() {
544        let mut lexer = Lexer::with_code(";");
545        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
546        let mut aliases = AliasSet::new();
547        aliases.insert(HashEntry::new(
548            ";".to_string(),
549            "x".to_string(),
550            false,
551            Location::dummy("?"),
552        ));
553        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
554
555        let result = parser.take_token_manual(true).now_or_never().unwrap();
556        let token = result.unwrap().unwrap();
557        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
558        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
559    }
560
561    #[test]
562    fn parser_take_token_manual_no_match() {
563        let mut lexer = Lexer::with_code("X");
564        let mut parser = Parser::new(&mut lexer);
565
566        let result = parser.take_token_manual(true).now_or_never().unwrap();
567        let token = result.unwrap().unwrap();
568        assert_eq!(token.to_string(), "X");
569    }
570
571    #[test]
572    fn parser_take_token_manual_recursive_substitution() {
573        let mut lexer = Lexer::with_code("X");
574        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
575        let mut aliases = AliasSet::new();
576        aliases.insert(HashEntry::new(
577            "X".to_string(),
578            "Y x".to_string(),
579            false,
580            Location::dummy("?"),
581        ));
582        aliases.insert(HashEntry::new(
583            "Y".to_string(),
584            "X y".to_string(),
585            false,
586            Location::dummy("?"),
587        ));
588        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
589
590        let result = parser.take_token_manual(true).now_or_never().unwrap();
591        assert_matches!(result, Ok(Rec::AliasSubstituted));
592
593        let result = parser.take_token_manual(true).now_or_never().unwrap();
594        assert_matches!(result, Ok(Rec::AliasSubstituted));
595
596        let result = parser.take_token_manual(true).now_or_never().unwrap();
597        let token = result.unwrap().unwrap();
598        assert_eq!(token.to_string(), "X");
599
600        let result = parser.take_token_manual(true).now_or_never().unwrap();
601        let token = result.unwrap().unwrap();
602        assert_eq!(token.to_string(), "y");
603
604        let rec = parser.take_token_manual(true).now_or_never().unwrap();
605        let token = rec.unwrap().unwrap();
606        assert_eq!(token.to_string(), "x");
607    }
608
609    #[test]
610    fn parser_take_token_manual_after_blank_ending_substitution() {
611        let mut lexer = Lexer::with_code("X\tY");
612        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
613        let mut aliases = AliasSet::new();
614        aliases.insert(HashEntry::new(
615            "X".to_string(),
616            " X ".to_string(),
617            false,
618            Location::dummy("?"),
619        ));
620        aliases.insert(HashEntry::new(
621            "Y".to_string(),
622            "y".to_string(),
623            false,
624            Location::dummy("?"),
625        ));
626        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
627
628        let result = parser.take_token_manual(true).now_or_never().unwrap();
629        assert_matches!(result, Ok(Rec::AliasSubstituted));
630
631        let result = parser.take_token_manual(true).now_or_never().unwrap();
632        let token = result.unwrap().unwrap();
633        assert_eq!(token.to_string(), "X");
634
635        let result = parser.take_token_manual(false).now_or_never().unwrap();
636        assert_matches!(result, Ok(Rec::AliasSubstituted));
637
638        let result = parser.take_token_manual(false).now_or_never().unwrap();
639        let token = result.unwrap().unwrap();
640        assert_eq!(token.to_string(), "y");
641    }
642
643    #[test]
644    fn parser_take_token_manual_not_after_blank_ending_substitution() {
645        let mut lexer = Lexer::with_code("X\tY");
646        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
647        let mut aliases = AliasSet::new();
648        aliases.insert(HashEntry::new(
649            "X".to_string(),
650            " X".to_string(),
651            false,
652            Location::dummy("?"),
653        ));
654        aliases.insert(HashEntry::new(
655            "Y".to_string(),
656            "y".to_string(),
657            false,
658            Location::dummy("?"),
659        ));
660        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
661
662        let result = parser.take_token_manual(true).now_or_never().unwrap();
663        assert_matches!(result, Ok(Rec::AliasSubstituted));
664
665        let result = parser.take_token_manual(true).now_or_never().unwrap();
666        let token = result.unwrap().unwrap();
667        assert_eq!(token.to_string(), "X");
668
669        let result = parser.take_token_manual(false).now_or_never().unwrap();
670        let token = result.unwrap().unwrap();
671        assert_eq!(token.to_string(), "Y");
672    }
673
674    #[test]
675    fn parser_take_token_manual_global() {
676        let mut lexer = Lexer::with_code("X");
677        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
678        let mut aliases = AliasSet::new();
679        aliases.insert(HashEntry::new(
680            "X".to_string(),
681            "x".to_string(),
682            true,
683            Location::dummy("?"),
684        ));
685        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
686
687        let result = parser.take_token_manual(false).now_or_never().unwrap();
688        assert_matches!(result, Ok(Rec::AliasSubstituted));
689
690        let result = parser.take_token_manual(false).now_or_never().unwrap();
691        let token = result.unwrap().unwrap();
692        assert_eq!(token.to_string(), "x");
693    }
694
695    #[test]
696    fn parser_take_token_auto_non_keyword() {
697        let mut lexer = Lexer::with_code("X");
698        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
699        let mut aliases = AliasSet::new();
700        aliases.insert(HashEntry::new(
701            "X".to_string(),
702            "x".to_string(),
703            true,
704            Location::dummy("?"),
705        ));
706        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
707
708        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
709        assert_eq!(token.to_string(), "x");
710    }
711
712    #[test]
713    fn parser_take_token_auto_keyword_matched() {
714        let mut lexer = Lexer::with_code("if");
715        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
716        let mut aliases = AliasSet::new();
717        aliases.insert(HashEntry::new(
718            "if".to_string(),
719            "x".to_string(),
720            true,
721            Location::dummy("?"),
722        ));
723        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
724
725        let token = parser
726            .take_token_auto(&[Keyword::If])
727            .now_or_never()
728            .unwrap()
729            .unwrap();
730        assert_eq!(token.to_string(), "if");
731    }
732
733    #[test]
734    fn parser_take_token_auto_keyword_unmatched() {
735        let mut lexer = Lexer::with_code("if");
736        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
737        let mut aliases = AliasSet::new();
738        aliases.insert(HashEntry::new(
739            "if".to_string(),
740            "x".to_string(),
741            true,
742            Location::dummy("?"),
743        ));
744        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
745
746        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
747        assert_eq!(token.to_string(), "x");
748    }
749
750    #[test]
751    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
752        let mut lexer = Lexer::with_code("X");
753        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
754        let mut aliases = AliasSet::new();
755        aliases.insert(HashEntry::new(
756            "X".to_string(),
757            "if".to_string(),
758            true,
759            Location::dummy("?"),
760        ));
761        aliases.insert(HashEntry::new(
762            "if".to_string(),
763            "x".to_string(),
764            true,
765            Location::dummy("?"),
766        ));
767        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
768
769        let token = parser
770            .take_token_auto(&[Keyword::If])
771            .now_or_never()
772            .unwrap()
773            .unwrap();
774        assert_eq!(token.to_string(), "if");
775    }
776
777    #[test]
778    fn parser_has_blank_true() {
779        let mut lexer = Lexer::with_code(" ");
780        let mut parser = Parser::new(&mut lexer);
781        let result = parser.has_blank().now_or_never().unwrap();
782        assert_eq!(result, Ok(true));
783    }
784
785    #[test]
786    fn parser_has_blank_false() {
787        let mut lexer = Lexer::with_code("(");
788        let mut parser = Parser::new(&mut lexer);
789        let result = parser.has_blank().now_or_never().unwrap();
790        assert_eq!(result, Ok(false));
791    }
792
793    #[test]
794    fn parser_has_blank_eof() {
795        let mut lexer = Lexer::with_code("");
796        let mut parser = Parser::new(&mut lexer);
797        let result = parser.has_blank().now_or_never().unwrap();
798        assert_eq!(result, Ok(false));
799    }
800
801    #[test]
802    fn parser_has_blank_true_with_line_continuations() {
803        let mut lexer = Lexer::with_code("\\\n\\\n ");
804        let mut parser = Parser::new(&mut lexer);
805        let result = parser.has_blank().now_or_never().unwrap();
806        assert_eq!(result, Ok(true));
807    }
808
809    #[test]
810    fn parser_has_blank_false_with_line_continuations() {
811        let mut lexer = Lexer::with_code("\\\n\\\n\\\n(");
812        let mut parser = Parser::new(&mut lexer);
813        let result = parser.has_blank().now_or_never().unwrap();
814        assert_eq!(result, Ok(false));
815    }
816
817    #[test]
818    #[should_panic(expected = "There should be no pending token")]
819    fn parser_has_blank_with_pending_token() {
820        let mut lexer = Lexer::with_code("foo");
821        let mut parser = Parser::new(&mut lexer);
822        parser.peek_token().now_or_never().unwrap().unwrap();
823        let _ = parser.has_blank().now_or_never().unwrap();
824    }
825
826    #[test]
827    fn parser_reading_no_here_doc_contents() {
828        let mut lexer = Lexer::with_code("X");
829        let mut parser = Parser::new(&mut lexer);
830        parser.here_doc_contents().now_or_never().unwrap().unwrap();
831
832        let location = lexer.location().now_or_never().unwrap().unwrap();
833        assert_eq!(location.code.start_line_number.get(), 1);
834        assert_eq!(location.range, 0..1);
835    }
836
837    #[test]
838    fn parser_reading_one_here_doc_content() {
839        let delimiter = "END".parse().unwrap();
840
841        let mut lexer = Lexer::with_code("END\nX");
842        let mut parser = Parser::new(&mut lexer);
843        let remove_tabs = false;
844        let here_doc = Rc::new(HereDoc {
845            delimiter,
846            remove_tabs,
847            content: OnceCell::new(),
848        });
849        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
850        parser.here_doc_contents().now_or_never().unwrap().unwrap();
851        assert_eq!(here_doc.delimiter.to_string(), "END");
852        assert_eq!(here_doc.remove_tabs, remove_tabs);
853        assert_eq!(here_doc.content.get().unwrap().0, []);
854
855        let location = lexer.location().now_or_never().unwrap().unwrap();
856        assert_eq!(location.code.start_line_number.get(), 1);
857        assert_eq!(location.range, 4..5);
858    }
859
860    #[test]
861    fn parser_reading_many_here_doc_contents() {
862        let delimiter1 = "ONE".parse().unwrap();
863        let delimiter2 = "TWO".parse().unwrap();
864        let delimiter3 = "THREE".parse().unwrap();
865
866        let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX");
867        let mut parser = Parser::new(&mut lexer);
868        let here_doc1 = Rc::new(HereDoc {
869            delimiter: delimiter1,
870            remove_tabs: false,
871            content: OnceCell::new(),
872        });
873        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
874        let here_doc2 = Rc::new(HereDoc {
875            delimiter: delimiter2,
876            remove_tabs: true,
877            content: OnceCell::new(),
878        });
879        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
880        let here_doc3 = Rc::new(HereDoc {
881            delimiter: delimiter3,
882            remove_tabs: false,
883            content: OnceCell::new(),
884        });
885        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
886        parser.here_doc_contents().now_or_never().unwrap().unwrap();
887        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
888        assert_eq!(here_doc1.remove_tabs, false);
889        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
890        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
891        assert_eq!(here_doc2.remove_tabs, true);
892        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
893        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
894        assert_eq!(here_doc3.remove_tabs, false);
895        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
896    }
897
898    #[test]
899    fn parser_reading_here_doc_contents_twice() {
900        let delimiter1 = "ONE".parse().unwrap();
901        let delimiter2 = "TWO".parse().unwrap();
902
903        let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n");
904        let mut parser = Parser::new(&mut lexer);
905        let here_doc1 = Rc::new(HereDoc {
906            delimiter: delimiter1,
907            remove_tabs: false,
908            content: OnceCell::new(),
909        });
910        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
911        parser.here_doc_contents().now_or_never().unwrap().unwrap();
912        let here_doc2 = Rc::new(HereDoc {
913            delimiter: delimiter2,
914            remove_tabs: true,
915            content: OnceCell::new(),
916        });
917        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
918        parser.here_doc_contents().now_or_never().unwrap().unwrap();
919        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
920        assert_eq!(here_doc1.remove_tabs, false);
921        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
922        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
923        assert_eq!(here_doc2.remove_tabs, true);
924        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
925    }
926
927    #[test]
928    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
929    fn parser_here_doc_contents_must_be_called_without_pending_token() {
930        let mut lexer = Lexer::with_code("X");
931        let mut parser = Parser::new(&mut lexer);
932        parser.peek_token().now_or_never().unwrap().unwrap();
933        parser.here_doc_contents().now_or_never().unwrap().unwrap();
934    }
935}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
