yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::syntax::HereDoc;
31use crate::syntax::MaybeLiteral as _;
32use crate::syntax::Word;
33use std::rc::Rc;
34
35/// Entire result of parsing
36pub type Result<T> = std::result::Result<T, Error>;
37
38/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
39/// parsing after alias substitution
40///
41/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
42///
43/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
44/// meaningful value and is returned from a parsing function that has performed alias substitution
45/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
46/// new source code produced by the substitution so that the syntax is correctly recognized in the
47/// new code.
48///
49/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
50/// recognized as an alias name during parse of a simple command, the simple command parser
51/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
52/// parser function, to recognize the `!` reserved word token as negation.
53///
54/// When a parser function successfully parses something, it returns the result in the `Parsed`
55/// variant. The caller then continues the remaining parse.
56#[derive(Copy, Clone, Debug, Eq, PartialEq)]
57pub enum Rec<T> {
58    /// Result of alias substitution
59    AliasSubstituted,
60    /// Successful parse result
61    Parsed(T),
62}
63
64impl<T> Rec<T> {
65    /// Tests if `self` is `AliasSubstituted`.
66    pub fn is_alias_substituted(&self) -> bool {
67        match self {
68            Rec::AliasSubstituted => true,
69            Rec::Parsed(_) => false,
70        }
71    }
72
73    /// Extracts the result of successful parsing.
74    ///
75    /// # Panics
76    ///
77    /// If `self` is `AliasSubstituted`.
78    pub fn unwrap(self) -> T {
79        match self {
80            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
81            Rec::Parsed(v) => v,
82        }
83    }
84
85    /// Transforms the result value in `self`.
86    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
87    where
88        F: FnOnce(T) -> Result<U>,
89    {
90        match self {
91            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
92            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
93        }
94    }
95}
96
97/// Set of parameters for constructing a [`Parser`]
98///
99/// `Config` is a builder for constructing a parser. A [new](Self::new)
100/// configuration starts with default settings. You can customize them by
101/// calling methods that can be chained. Finally, you can create a parser by
102/// providing the lexer to the [`input`](Self::input) method.
103///
104/// Note that this struct is for configuring a parser. To configure the
105/// [`Lexer`], use [`yash_env::parser::Config`].
106#[derive(Debug)]
107#[must_use = "Config must be used to create a parser"]
108pub struct Config<'a> {
109    /// Collection of aliases the parser applies to substitute command words
110    aliases: &'a dyn crate::alias::Glossary,
111
112    /// Glossary that determines whether a command name is a declaration utility
113    decl_utils: &'a dyn crate::decl_util::Glossary,
114}
115
116impl<'a> Config<'a> {
117    /// Creates a new configuration with default settings.
118    ///
119    /// You can also call [`Parser::config`] to create a new configuration.
120    pub fn new() -> Self {
121        Self {
122            aliases: &crate::alias::EmptyGlossary,
123            decl_utils: &crate::decl_util::PosixGlossary,
124        }
125    }
126
127    /// Sets the glossary of aliases.
128    ///
129    /// The parser uses the glossary to look up aliases and substitute command
130    /// words. The default glossary is [empty](crate::alias::EmptyGlossary).
131    #[inline]
132    pub fn aliases(&mut self, aliases: &'a dyn Glossary) -> &mut Self {
133        self.aliases = aliases;
134        self
135    }
136
137    /// Sets the glossary of declaration utilities.
138    ///
139    /// The parser uses the glossary to determine whether a command name is a
140    /// declaration utility. The default glossary is [`PosixGlossary`], which
141    /// recognizes the declaration utilities defined by POSIX. You can make
142    /// arbitrary command names declaration utilities by providing a custom
143    /// glossary. To meet the POSIX standard, the glossary's
144    /// [`is_declaration_utility`] method must return:
145    ///
146    /// - `Some(true)` for `export` and `readonly`
147    /// - `None` for `command`
148    ///
149    /// For detailed information on declaration utilities, see the
150    /// [`decl_utils`] module.
151    ///
152    /// [`decl_utils`]: crate::decl_util
153    /// [`PosixGlossary`]: crate::decl_util::PosixGlossary
154    /// [`is_declaration_utility`]: crate::decl_util::Glossary::is_declaration_utility
155    #[inline]
156    pub fn declaration_utilities(
157        &mut self,
158        decl_utils: &'a dyn crate::decl_util::Glossary,
159    ) -> &mut Self {
160        self.decl_utils = decl_utils;
161        self
162    }
163
164    /// Creates a parser with the given lexer.
165    pub fn input<'b>(&self, lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
166        Parser {
167            lexer,
168            aliases: self.aliases,
169            decl_utils: self.decl_utils,
170            token: None,
171            unread_here_docs: Vec::new(),
172        }
173    }
174}
175
176impl Default for Config<'_> {
177    fn default() -> Self {
178        Self::new()
179    }
180}
181
182/// The shell syntax parser
183///
184/// A parser manages a set of data used in syntax parsing. It keeps a reference
185/// to a [lexer](Lexer) that provides tokens to parse. It also has some
186/// parameters that can be set by a [configuration](Config) and affect the
187/// parsing process.
188///
189/// The [`new`](Self::new) function directly creates a parser with default
190/// settings. If you want to customize the settings, you can use the
191/// [`config`](Self::config) function to create a configuration and then create a
192/// parser with the configuration.
193///
194/// # Parsing here-documents
195///
196/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
197/// with empty content. The parser creates the `HereDoc` instance when it finds
198/// a here-document operator, but it has not read its content at that time. When
199/// finding a newline token, the parser reads the content and fills it into the
200/// `HereDoc` instance.
201///
202/// Unless you are interested in parsing a specific syntactic construct that is
203/// only part of source code, you will want to use a function that returns a
204/// complete result filled with proper here-document contents if any.
205/// Then the [`command_line`](Self::command_line) function is for you.
206/// See also the [module documentation](super).
207#[derive(Debug)]
208#[must_use = "Parser must be used to parse syntax"]
209pub struct Parser<'a, 'b> {
210    /// Lexer that provides tokens
211    lexer: &'a mut Lexer<'b>,
212
213    /// Collection of aliases the parser applies to substitute command words
214    aliases: &'a dyn crate::alias::Glossary,
215
216    /// Glossary that determines whether a command name is a declaration utility
217    decl_utils: &'a dyn crate::decl_util::Glossary,
218
219    /// Token to parse next
220    ///
221    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
222    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
223    token: Option<Result<Token>>,
224
225    /// Here-documents without contents
226    ///
227    /// The here-document is added to this list when the parser finds a
228    /// here-document operator. After consuming the next newline token, the
229    /// parser reads and fills the contents, then clears this list.
230    unread_here_docs: Vec<Rc<HereDoc>>,
231}
232
233impl<'a, 'b> Parser<'a, 'b> {
234    /// Creates a new configuration with default settings.
235    ///
236    /// This is a synonym for [`Config::new`]. Customize the settings by calling
237    /// methods of the returned configuration and then create a parser by calling
238    /// its [`input`](Config::input) method.
239    #[inline(always)]
240    pub fn config() -> Config<'a> {
241        Config::new()
242    }
243
244    /// Creates a new parser based on the given lexer.
245    ///
246    /// The parser uses the lexer to read tokens. All other settings are default.
247    /// To customize the settings, use the [`config`](Self::config) function.
248    pub fn new(lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
249        Self::config().input(lexer)
250    }
251
252    /// Reads a next token if the current token is `None`.
253    async fn require_token(&mut self) {
254        #[allow(clippy::question_mark, reason = "false positive")]
255        // TODO https://github.com/rust-lang/rust-clippy/issues/9518
256        if self.token.is_none() {
257            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
258                Err(e)
259            } else {
260                self.lexer.token().await
261            });
262        }
263    }
264
265    /// Returns the parsing mode of the underlying lexer.
266    ///
267    /// The parser consults the mode to decide which syntax to accept. See
268    /// [`Lexer::mode`](crate::parser::lex::Lexer::mode).
269    pub(super) fn mode(&self) -> yash_env::parser::Mode {
270        self.lexer.mode()
271    }
272
273    /// Returns a reference to the current token.
274    ///
275    /// If the current token is not yet read from the underlying lexer, it is read.
276    pub async fn peek_token(&mut self) -> Result<&Token> {
277        self.require_token().await;
278        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
279    }
280
281    /// Consumes the current token without performing alias substitution.
282    ///
283    /// If the current token is not yet read from the underlying lexer, it is read.
284    ///
285    /// This function does not perform alias substitution and therefore should be
286    /// used only in context where no alias substitution is expected. Otherwise,
287    /// you should use [`take_token_manual`](Self::take_token_manual) or
288    /// [`take_token_auto`](Self::take_token_auto) instead.
289    pub async fn take_token_raw(&mut self) -> Result<Token> {
290        self.require_token().await;
291        self.token.take().unwrap()
292    }
293
294    /// Performs alias substitution on a token that has just been
295    /// [taken](Self::take_token_raw).
296    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
297        // TODO Only POSIXly-valid alias name should be recognized in POSIXly-correct mode.
298        if !self.aliases.is_empty()
299            && let Token(_) = token.id
300            && let Some(name) = token.word.to_string_if_literal()
301            && !token.word.location.code.source.is_alias_for(&name)
302            && let Some(alias) = self.aliases.look_up(&name)
303            && (is_command_name
304                || alias.global
305                || self.lexer.is_after_blank_ending_alias(token.index))
306        {
307            self.lexer.substitute_alias(token.index, &alias);
308            return Rec::AliasSubstituted;
309        }
310
311        Rec::Parsed(token)
312    }
313
314    /// Consumes the current token after performing applicable alias substitution.
315    ///
316    /// If the current token is not yet read from the underlying lexer, it is read.
317    ///
318    /// This function checks if the token is the name of an alias. If it is,
319    /// alias substitution is performed on the token and the result is
320    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
321    ///
322    /// Alias substitution is performed only if at least one of the following is
323    /// true:
324    ///
325    /// - The token is the first command word in a simple command, that is, it is
326    ///   the word for the command name. (This condition should be specified by the
327    ///   `is_command_name` parameter.)
328    /// - The token comes just after the replacement string of another alias
329    ///   substitution that ends with a blank character.
330    /// - The token names a global alias.
331    ///
332    /// However, alias substitution should _not_ be performed on a reserved word
333    /// in any case. It is your responsibility to check the token type and not to
334    /// call this function on a reserved word. That is why this function is named
335    /// `manual`. To consume a reserved word without performing alias
336    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
337    /// [`take_token_auto`](Self::take_token_auto).
338    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
339        let token = self.take_token_raw().await?;
340        Ok(self.substitute_alias(token, is_command_name))
341    }
342
343    /// Consumes the current token after performing applicable alias substitution.
344    ///
345    /// This function performs alias substitution unless the result is one of the
346    /// reserved words specified in the argument.
347    ///
348    /// Alias substitution is performed repeatedly until a non-alias token is
349    /// found. That is why this function is named `auto`. This function should be
350    /// used only in contexts where no backtrack is needed after alias
351    /// substitution. If you need to backtrack or want to know whether alias
352    /// substitution was performed or not, you should use
353    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
354    /// alias substitution at most once and returns `Rec`.
355    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
356        loop {
357            let token = self.take_token_raw().await?;
358            if let Token(Some(keyword)) = token.id
359                && keywords.contains(&keyword)
360            {
361                return Ok(token);
362            }
363            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
364                return Ok(token);
365            }
366        }
367    }
368
369    /// Tests if there is a blank before the next token.
370    ///
371    /// This function can be called to tell whether the previous and next tokens
372    /// are separated by a blank or they are adjacent.
373    ///
374    /// This function must be called after the previous token has been taken (by
375    /// one of [`take_token_raw`](Self::take_token_raw),
376    /// [`take_token_manual`](Self::take_token_manual) and
377    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
378    /// [peeked](Self::peek_token). Otherwise, this function would panic.
379    ///
380    /// # Panics
381    ///
382    /// If the previous token has not been taken or the next token has been
383    /// peeked.
384    pub async fn has_blank(&mut self) -> Result<bool> {
385        assert!(self.token.is_none(), "There should be no pending token");
386        let c = self.lexer.peek_char().await?;
387        Ok(c.is_some_and(is_blank))
388    }
389
390    /// Remembers the given partial here-document for later parsing of its content.
391    ///
392    /// The remembered here-document's content will be parsed when
393    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
394    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
395        self.unread_here_docs.push(here_doc)
396    }
397
398    /// Reads here-document contents that matches the remembered list of
399    /// here-document operators.
400    ///
401    /// This function reads here-document contents corresponding to
402    /// here-document operators that have been saved with
403    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
404    /// The results are inserted to the `content` field of the `HereDoc`
405    /// instances.
406    ///
407    /// This function must be called just after a newline token has been taken
408    /// (either [manual](Self::take_token_manual) or
409    /// [auto](Self::take_token_auto)). If there is a pending token that has been
410    /// peeked but not yet taken, this function will panic!
411    pub async fn here_doc_contents(&mut self) -> Result<()> {
412        assert!(
413            self.token.is_none(),
414            "No token must be peeked before reading here-doc contents"
415        );
416
417        for here_doc in self.unread_here_docs.drain(..) {
418            self.lexer.here_doc_content(&here_doc).await?;
419        }
420
421        Ok(())
422    }
423
424    /// Ensures that there is no pending partial here-document.
425    ///
426    /// If there is any, this function returns a `MissingHereDocContent` error.
427    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
428        match self.unread_here_docs.first() {
429            None => Ok(()),
430            Some(here_doc) => Err(Error {
431                cause: SyntaxError::MissingHereDocContent.into(),
432                location: here_doc.delimiter.location.clone(),
433            }),
434        }
435    }
436
437    /// Determines whether a word names a declaration utility.
438    ///
439    /// See [`decl_utils`](crate::decl_util) for more information.
440    pub(super) fn word_names_declaration_utility(&self, word: &Word) -> Option<bool> {
441        if let Some(name) = word.to_string_if_literal() {
442            self.decl_utils.is_declaration_utility(&name)
443        } else {
444            Some(false)
445        }
446    }
447}
448
449#[allow(
450    clippy::bool_assert_comparison,
451    reason = "to make the expected values clearer"
452)]
453#[cfg(test)]
454mod tests {
455    use super::*;
456    use crate::alias::AliasSet;
457    use crate::alias::HashEntry;
458    use crate::source::Location;
459    use futures_util::FutureExt as _;
460    use std::assert_matches;
461    use std::cell::OnceCell;
462
463    #[test]
464    fn parser_take_token_manual_successful_substitution() {
465        let mut lexer = Lexer::with_code("X");
466        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
467        let mut aliases = AliasSet::new();
468        aliases.insert(HashEntry::new(
469            "X".to_string(),
470            "x".to_string(),
471            false,
472            Location::dummy("?"),
473        ));
474        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
475
476        let result = parser.take_token_manual(true).now_or_never().unwrap();
477        assert_matches!(result, Ok(Rec::AliasSubstituted));
478
479        let result = parser.take_token_manual(true).now_or_never().unwrap();
480        let token = result.unwrap().unwrap();
481        assert_eq!(token.to_string(), "x");
482    }
483
484    #[test]
485    fn parser_take_token_manual_not_command_name() {
486        let mut lexer = Lexer::with_code("X");
487        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
488        let mut aliases = AliasSet::new();
489        aliases.insert(HashEntry::new(
490            "X".to_string(),
491            "x".to_string(),
492            false,
493            Location::dummy("?"),
494        ));
495        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
496
497        let result = parser.take_token_manual(false).now_or_never().unwrap();
498        let token = result.unwrap().unwrap();
499        assert_eq!(token.to_string(), "X");
500    }
501
502    #[test]
503    fn parser_take_token_manual_not_literal() {
504        let mut lexer = Lexer::with_code(r"\X");
505        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
506        let mut aliases = AliasSet::new();
507        aliases.insert(HashEntry::new(
508            "X".to_string(),
509            "x".to_string(),
510            false,
511            Location::dummy("?"),
512        ));
513        aliases.insert(HashEntry::new(
514            r"\X".to_string(),
515            "quoted".to_string(),
516            false,
517            Location::dummy("?"),
518        ));
519        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
520
521        let result = parser.take_token_manual(true).now_or_never().unwrap();
522        let token = result.unwrap().unwrap();
523        assert_eq!(token.to_string(), r"\X");
524    }
525
526    #[test]
527    fn parser_take_token_manual_operator() {
528        let mut lexer = Lexer::with_code(";");
529        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
530        let mut aliases = AliasSet::new();
531        aliases.insert(HashEntry::new(
532            ";".to_string(),
533            "x".to_string(),
534            false,
535            Location::dummy("?"),
536        ));
537        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
538
539        let result = parser.take_token_manual(true).now_or_never().unwrap();
540        let token = result.unwrap().unwrap();
541        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
542        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
543    }
544
545    #[test]
546    fn parser_take_token_manual_no_match() {
547        let mut lexer = Lexer::with_code("X");
548        let mut parser = Parser::new(&mut lexer);
549
550        let result = parser.take_token_manual(true).now_or_never().unwrap();
551        let token = result.unwrap().unwrap();
552        assert_eq!(token.to_string(), "X");
553    }
554
555    #[test]
556    fn parser_take_token_manual_recursive_substitution() {
557        let mut lexer = Lexer::with_code("X");
558        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
559        let mut aliases = AliasSet::new();
560        aliases.insert(HashEntry::new(
561            "X".to_string(),
562            "Y x".to_string(),
563            false,
564            Location::dummy("?"),
565        ));
566        aliases.insert(HashEntry::new(
567            "Y".to_string(),
568            "X y".to_string(),
569            false,
570            Location::dummy("?"),
571        ));
572        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
573
574        let result = parser.take_token_manual(true).now_or_never().unwrap();
575        assert_matches!(result, Ok(Rec::AliasSubstituted));
576
577        let result = parser.take_token_manual(true).now_or_never().unwrap();
578        assert_matches!(result, Ok(Rec::AliasSubstituted));
579
580        let result = parser.take_token_manual(true).now_or_never().unwrap();
581        let token = result.unwrap().unwrap();
582        assert_eq!(token.to_string(), "X");
583
584        let result = parser.take_token_manual(true).now_or_never().unwrap();
585        let token = result.unwrap().unwrap();
586        assert_eq!(token.to_string(), "y");
587
588        let rec = parser.take_token_manual(true).now_or_never().unwrap();
589        let token = rec.unwrap().unwrap();
590        assert_eq!(token.to_string(), "x");
591    }
592
593    #[test]
594    fn parser_take_token_manual_after_blank_ending_substitution() {
595        let mut lexer = Lexer::with_code("X\tY");
596        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
597        let mut aliases = AliasSet::new();
598        aliases.insert(HashEntry::new(
599            "X".to_string(),
600            " X ".to_string(),
601            false,
602            Location::dummy("?"),
603        ));
604        aliases.insert(HashEntry::new(
605            "Y".to_string(),
606            "y".to_string(),
607            false,
608            Location::dummy("?"),
609        ));
610        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
611
612        let result = parser.take_token_manual(true).now_or_never().unwrap();
613        assert_matches!(result, Ok(Rec::AliasSubstituted));
614
615        let result = parser.take_token_manual(true).now_or_never().unwrap();
616        let token = result.unwrap().unwrap();
617        assert_eq!(token.to_string(), "X");
618
619        let result = parser.take_token_manual(false).now_or_never().unwrap();
620        assert_matches!(result, Ok(Rec::AliasSubstituted));
621
622        let result = parser.take_token_manual(false).now_or_never().unwrap();
623        let token = result.unwrap().unwrap();
624        assert_eq!(token.to_string(), "y");
625    }
626
627    #[test]
628    fn parser_take_token_manual_not_after_blank_ending_substitution() {
629        let mut lexer = Lexer::with_code("X\tY");
630        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
631        let mut aliases = AliasSet::new();
632        aliases.insert(HashEntry::new(
633            "X".to_string(),
634            " X".to_string(),
635            false,
636            Location::dummy("?"),
637        ));
638        aliases.insert(HashEntry::new(
639            "Y".to_string(),
640            "y".to_string(),
641            false,
642            Location::dummy("?"),
643        ));
644        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
645
646        let result = parser.take_token_manual(true).now_or_never().unwrap();
647        assert_matches!(result, Ok(Rec::AliasSubstituted));
648
649        let result = parser.take_token_manual(true).now_or_never().unwrap();
650        let token = result.unwrap().unwrap();
651        assert_eq!(token.to_string(), "X");
652
653        let result = parser.take_token_manual(false).now_or_never().unwrap();
654        let token = result.unwrap().unwrap();
655        assert_eq!(token.to_string(), "Y");
656    }
657
658    #[test]
659    fn parser_take_token_manual_global() {
660        let mut lexer = Lexer::with_code("X");
661        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
662        let mut aliases = AliasSet::new();
663        aliases.insert(HashEntry::new(
664            "X".to_string(),
665            "x".to_string(),
666            true,
667            Location::dummy("?"),
668        ));
669        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
670
671        let result = parser.take_token_manual(false).now_or_never().unwrap();
672        assert_matches!(result, Ok(Rec::AliasSubstituted));
673
674        let result = parser.take_token_manual(false).now_or_never().unwrap();
675        let token = result.unwrap().unwrap();
676        assert_eq!(token.to_string(), "x");
677    }
678
679    #[test]
680    fn parser_take_token_auto_non_keyword() {
681        let mut lexer = Lexer::with_code("X");
682        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
683        let mut aliases = AliasSet::new();
684        aliases.insert(HashEntry::new(
685            "X".to_string(),
686            "x".to_string(),
687            true,
688            Location::dummy("?"),
689        ));
690        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
691
692        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
693        assert_eq!(token.to_string(), "x");
694    }
695
696    #[test]
697    fn parser_take_token_auto_keyword_matched() {
698        let mut lexer = Lexer::with_code("if");
699        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
700        let mut aliases = AliasSet::new();
701        aliases.insert(HashEntry::new(
702            "if".to_string(),
703            "x".to_string(),
704            true,
705            Location::dummy("?"),
706        ));
707        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
708
709        let token = parser
710            .take_token_auto(&[Keyword::If])
711            .now_or_never()
712            .unwrap()
713            .unwrap();
714        assert_eq!(token.to_string(), "if");
715    }
716
717    #[test]
718    fn parser_take_token_auto_keyword_unmatched() {
719        let mut lexer = Lexer::with_code("if");
720        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
721        let mut aliases = AliasSet::new();
722        aliases.insert(HashEntry::new(
723            "if".to_string(),
724            "x".to_string(),
725            true,
726            Location::dummy("?"),
727        ));
728        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
729
730        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
731        assert_eq!(token.to_string(), "x");
732    }
733
734    #[test]
735    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
736        let mut lexer = Lexer::with_code("X");
737        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
738        let mut aliases = AliasSet::new();
739        aliases.insert(HashEntry::new(
740            "X".to_string(),
741            "if".to_string(),
742            true,
743            Location::dummy("?"),
744        ));
745        aliases.insert(HashEntry::new(
746            "if".to_string(),
747            "x".to_string(),
748            true,
749            Location::dummy("?"),
750        ));
751        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
752
753        let token = parser
754            .take_token_auto(&[Keyword::If])
755            .now_or_never()
756            .unwrap()
757            .unwrap();
758        assert_eq!(token.to_string(), "if");
759    }
760
761    #[test]
762    fn parser_has_blank_true() {
763        let mut lexer = Lexer::with_code(" ");
764        let mut parser = Parser::new(&mut lexer);
765        let result = parser.has_blank().now_or_never().unwrap();
766        assert_eq!(result, Ok(true));
767    }
768
769    #[test]
770    fn parser_has_blank_false() {
771        let mut lexer = Lexer::with_code("(");
772        let mut parser = Parser::new(&mut lexer);
773        let result = parser.has_blank().now_or_never().unwrap();
774        assert_eq!(result, Ok(false));
775    }
776
777    #[test]
778    fn parser_has_blank_eof() {
779        let mut lexer = Lexer::with_code("");
780        let mut parser = Parser::new(&mut lexer);
781        let result = parser.has_blank().now_or_never().unwrap();
782        assert_eq!(result, Ok(false));
783    }
784
785    #[test]
786    fn parser_has_blank_true_with_line_continuations() {
787        let mut lexer = Lexer::with_code("\\\n\\\n ");
788        let mut parser = Parser::new(&mut lexer);
789        let result = parser.has_blank().now_or_never().unwrap();
790        assert_eq!(result, Ok(true));
791    }
792
793    #[test]
794    fn parser_has_blank_false_with_line_continuations() {
795        let mut lexer = Lexer::with_code("\\\n\\\n\\\n(");
796        let mut parser = Parser::new(&mut lexer);
797        let result = parser.has_blank().now_or_never().unwrap();
798        assert_eq!(result, Ok(false));
799    }
800
801    #[test]
802    #[should_panic(expected = "There should be no pending token")]
803    fn parser_has_blank_with_pending_token() {
804        let mut lexer = Lexer::with_code("foo");
805        let mut parser = Parser::new(&mut lexer);
806        parser.peek_token().now_or_never().unwrap().unwrap();
807        let _ = parser.has_blank().now_or_never().unwrap();
808    }
809
810    #[test]
811    fn parser_reading_no_here_doc_contents() {
812        let mut lexer = Lexer::with_code("X");
813        let mut parser = Parser::new(&mut lexer);
814        parser.here_doc_contents().now_or_never().unwrap().unwrap();
815
816        let location = lexer.location().now_or_never().unwrap().unwrap();
817        assert_eq!(location.code.start_line_number.get(), 1);
818        assert_eq!(location.range, 0..1);
819    }
820
821    #[test]
822    fn parser_reading_one_here_doc_content() {
823        let delimiter = "END".parse().unwrap();
824
825        let mut lexer = Lexer::with_code("END\nX");
826        let mut parser = Parser::new(&mut lexer);
827        let remove_tabs = false;
828        let here_doc = Rc::new(HereDoc {
829            delimiter,
830            remove_tabs,
831            content: OnceCell::new(),
832        });
833        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
834        parser.here_doc_contents().now_or_never().unwrap().unwrap();
835        assert_eq!(here_doc.delimiter.to_string(), "END");
836        assert_eq!(here_doc.remove_tabs, remove_tabs);
837        assert_eq!(here_doc.content.get().unwrap().0, []);
838
839        let location = lexer.location().now_or_never().unwrap().unwrap();
840        assert_eq!(location.code.start_line_number.get(), 1);
841        assert_eq!(location.range, 4..5);
842    }
843
844    #[test]
845    fn parser_reading_many_here_doc_contents() {
846        let delimiter1 = "ONE".parse().unwrap();
847        let delimiter2 = "TWO".parse().unwrap();
848        let delimiter3 = "THREE".parse().unwrap();
849
850        let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX");
851        let mut parser = Parser::new(&mut lexer);
852        let here_doc1 = Rc::new(HereDoc {
853            delimiter: delimiter1,
854            remove_tabs: false,
855            content: OnceCell::new(),
856        });
857        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
858        let here_doc2 = Rc::new(HereDoc {
859            delimiter: delimiter2,
860            remove_tabs: true,
861            content: OnceCell::new(),
862        });
863        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
864        let here_doc3 = Rc::new(HereDoc {
865            delimiter: delimiter3,
866            remove_tabs: false,
867            content: OnceCell::new(),
868        });
869        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
870        parser.here_doc_contents().now_or_never().unwrap().unwrap();
871        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
872        assert_eq!(here_doc1.remove_tabs, false);
873        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
874        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
875        assert_eq!(here_doc2.remove_tabs, true);
876        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
877        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
878        assert_eq!(here_doc3.remove_tabs, false);
879        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
880    }
881
882    #[test]
883    fn parser_reading_here_doc_contents_twice() {
884        let delimiter1 = "ONE".parse().unwrap();
885        let delimiter2 = "TWO".parse().unwrap();
886
887        let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n");
888        let mut parser = Parser::new(&mut lexer);
889        let here_doc1 = Rc::new(HereDoc {
890            delimiter: delimiter1,
891            remove_tabs: false,
892            content: OnceCell::new(),
893        });
894        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
895        parser.here_doc_contents().now_or_never().unwrap().unwrap();
896        let here_doc2 = Rc::new(HereDoc {
897            delimiter: delimiter2,
898            remove_tabs: true,
899            content: OnceCell::new(),
900        });
901        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
902        parser.here_doc_contents().now_or_never().unwrap().unwrap();
903        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
904        assert_eq!(here_doc1.remove_tabs, false);
905        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
906        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
907        assert_eq!(here_doc2.remove_tabs, true);
908        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
909    }
910
911    #[test]
912    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
913    fn parser_here_doc_contents_must_be_called_without_pending_token() {
914        let mut lexer = Lexer::with_code("X");
915        let mut parser = Parser::new(&mut lexer);
916        parser.peek_token().now_or_never().unwrap().unwrap();
917        parser.here_doc_contents().now_or_never().unwrap().unwrap();
918    }
919}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
