yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::syntax::HereDoc;
31use crate::syntax::MaybeLiteral as _;
32use crate::syntax::Word;
33use std::rc::Rc;
34
35/// Entire result of parsing
36pub type Result<T> = std::result::Result<T, Error>;
37
38/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
39/// parsing after alias substitution
40///
41/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
42///
43/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
44/// meaningful value and is returned from a parsing function that has performed alias substitution
45/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
46/// new source code produced by the substitution so that the syntax is correctly recognized in the
47/// new code.
48///
49/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
50/// recognized as an alias name during parse of a simple command, the simple command parser
51/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
52/// parser function, to recognize the `!` reserved word token as negation.
53///
54/// When a parser function successfully parses something, it returns the result in the `Parsed`
55/// variant. The caller then continues the remaining parse.
56#[derive(Copy, Clone, Debug, Eq, PartialEq)]
57pub enum Rec<T> {
58    /// Result of alias substitution
59    AliasSubstituted,
60    /// Successful parse result
61    Parsed(T),
62}
63
64impl<T> Rec<T> {
65    /// Tests if `self` is `AliasSubstituted`.
66    pub fn is_alias_substituted(&self) -> bool {
67        match self {
68            Rec::AliasSubstituted => true,
69            Rec::Parsed(_) => false,
70        }
71    }
72
73    /// Extracts the result of successful parsing.
74    ///
75    /// # Panics
76    ///
77    /// If `self` is `AliasSubstituted`.
78    pub fn unwrap(self) -> T {
79        match self {
80            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
81            Rec::Parsed(v) => v,
82        }
83    }
84
85    /// Transforms the result value in `self`.
86    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
87    where
88        F: FnOnce(T) -> Result<U>,
89    {
90        match self {
91            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
92            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
93        }
94    }
95}
96
97/// Set of parameters for constructing a [`Parser`]
98///
99/// `Config` is a builder for constructing a parser. A [new](Self::new)
100/// configuration starts with default settings. You can customize them by
101/// calling methods that can be chained. Finally, you can create a parser by
102/// providing the lexer to the [`input`](Self::input) method.
103///
104/// Note that this struct is for configuring a parser. To configure the
105/// [`Lexer`], use [`yash_env::parser::Config`].
106#[derive(Debug)]
107#[must_use = "Config must be used to create a parser"]
108pub struct Config<'a> {
109    /// Collection of aliases the parser applies to substitute command words
110    aliases: &'a dyn crate::alias::Glossary,
111
112    /// Glossary that determines whether a command name is a declaration utility
113    decl_utils: &'a dyn crate::decl_util::Glossary,
114}
115
116impl<'a> Config<'a> {
117    /// Creates a new configuration with default settings.
118    ///
119    /// You can also call [`Parser::config`] to create a new configuration.
120    pub fn new() -> Self {
121        Self {
122            aliases: &crate::alias::EmptyGlossary,
123            decl_utils: &crate::decl_util::PosixGlossary,
124        }
125    }
126
127    /// Sets the glossary of aliases.
128    ///
129    /// The parser uses the glossary to look up aliases and substitute command
130    /// words. The default glossary is [empty](crate::alias::EmptyGlossary).
131    #[inline]
132    pub fn aliases(&mut self, aliases: &'a dyn Glossary) -> &mut Self {
133        self.aliases = aliases;
134        self
135    }
136
137    /// Sets the glossary of declaration utilities.
138    ///
139    /// The parser uses the glossary to determine whether a command name is a
140    /// declaration utility. The default glossary is [`PosixGlossary`], which
141    /// recognizes the declaration utilities defined by POSIX. You can make
142    /// arbitrary command names declaration utilities by providing a custom
143    /// glossary. To meet the POSIX standard, the glossary's
144    /// [`is_declaration_utility`] method must return:
145    ///
146    /// - `Some(true)` for `export` and `readonly`
147    /// - `None` for `command`
148    ///
149    /// For detailed information on declaration utilities, see the
150    /// [`decl_utils`] module.
151    ///
152    /// [`decl_utils`]: crate::decl_util
153    /// [`PosixGlossary`]: crate::decl_util::PosixGlossary
154    /// [`is_declaration_utility`]: crate::decl_util::Glossary::is_declaration_utility
155    #[inline]
156    pub fn declaration_utilities(
157        &mut self,
158        decl_utils: &'a dyn crate::decl_util::Glossary,
159    ) -> &mut Self {
160        self.decl_utils = decl_utils;
161        self
162    }
163
164    /// Creates a parser with the given lexer.
165    pub fn input<'b>(&self, lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
166        Parser {
167            lexer,
168            aliases: self.aliases,
169            decl_utils: self.decl_utils,
170            token: None,
171            unread_here_docs: Vec::new(),
172        }
173    }
174}
175
176impl Default for Config<'_> {
177    fn default() -> Self {
178        Self::new()
179    }
180}
181
182/// The shell syntax parser
183///
184/// A parser manages a set of data used in syntax parsing. It keeps a reference
185/// to a [lexer](Lexer) that provides tokens to parse. It also has some
186/// parameters that can be set by a [configuration](Config) and affect the
187/// parsing process.
188///
189/// The [`new`](Self::new) function directly creates a parser with default
190/// settings. If you want to customize the settings, you can use the
191/// [`config`](Self::config) function to create a configuration and then create a
192/// parser with the configuration.
193///
194/// # Parsing here-documents
195///
196/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
197/// with empty content. The parser creates the `HereDoc` instance when it finds
198/// a here-document operator, but it has not read its content at that time. When
199/// finding a newline token, the parser reads the content and fills it into the
200/// `HereDoc` instance.
201///
202/// Unless you are interested in parsing a specific syntactic construct that is
203/// only part of source code, you will want to use a function that returns a
204/// complete result filled with proper here-document contents if any.
205/// Then the [`command_line`](Self::command_line) function is for you.
206/// See also the [module documentation](super).
207#[derive(Debug)]
208#[must_use = "Parser must be used to parse syntax"]
209pub struct Parser<'a, 'b> {
210    /// Lexer that provides tokens
211    lexer: &'a mut Lexer<'b>,
212
213    /// Collection of aliases the parser applies to substitute command words
214    aliases: &'a dyn crate::alias::Glossary,
215
216    /// Glossary that determines whether a command name is a declaration utility
217    decl_utils: &'a dyn crate::decl_util::Glossary,
218
219    /// Token to parse next
220    ///
221    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
222    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
223    token: Option<Result<Token>>,
224
225    /// Here-documents without contents
226    ///
227    /// The here-document is added to this list when the parser finds a
228    /// here-document operator. After consuming the next newline token, the
229    /// parser reads and fills the contents, then clears this list.
230    unread_here_docs: Vec<Rc<HereDoc>>,
231}
232
233impl<'a, 'b> Parser<'a, 'b> {
234    /// Creates a new configuration with default settings.
235    ///
236    /// This is a synonym for [`Config::new`]. Customize the settings by calling
237    /// methods of the returned configuration and then create a parser by calling
238    /// its [`input`](Config::input) method.
239    #[inline(always)]
240    pub fn config() -> Config<'a> {
241        Config::new()
242    }
243
244    /// Creates a new parser based on the given lexer.
245    ///
246    /// The parser uses the lexer to read tokens. All other settings are default.
247    /// To customize the settings, use the [`config`](Self::config) function.
248    pub fn new(lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
249        Self::config().input(lexer)
250    }
251
252    /// Reads a next token if the current token is `None`.
253    async fn require_token(&mut self) {
254        #[allow(clippy::question_mark, reason = "false positive")]
255        // TODO https://github.com/rust-lang/rust-clippy/issues/9518
256        if self.token.is_none() {
257            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
258                Err(e)
259            } else {
260                self.lexer.token().await
261            });
262        }
263    }
264
265    /// Returns a reference to the current token.
266    ///
267    /// If the current token is not yet read from the underlying lexer, it is read.
268    pub async fn peek_token(&mut self) -> Result<&Token> {
269        self.require_token().await;
270        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
271    }
272
273    /// Consumes the current token without performing alias substitution.
274    ///
275    /// If the current token is not yet read from the underlying lexer, it is read.
276    ///
277    /// This function does not perform alias substitution and therefore should be
278    /// used only in context where no alias substitution is expected. Otherwise,
279    /// you should use [`take_token_manual`](Self::take_token_manual) or
280    /// [`take_token_auto`](Self::take_token_auto) instead.
281    pub async fn take_token_raw(&mut self) -> Result<Token> {
282        self.require_token().await;
283        self.token.take().unwrap()
284    }
285
286    /// Performs alias substitution on a token that has just been
287    /// [taken](Self::take_token_raw).
288    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
289        // TODO Only POSIXly-valid alias name should be recognized in POSIXly-correct mode.
290        if !self.aliases.is_empty() {
291            if let Token(_) = token.id {
292                if let Some(name) = token.word.to_string_if_literal() {
293                    if !token.word.location.code.source.is_alias_for(&name) {
294                        if let Some(alias) = self.aliases.look_up(&name) {
295                            if is_command_name
296                                || alias.global
297                                || self.lexer.is_after_blank_ending_alias(token.index)
298                            {
299                                self.lexer.substitute_alias(token.index, &alias);
300                                return Rec::AliasSubstituted;
301                            }
302                        }
303                    }
304                }
305            }
306        }
307
308        Rec::Parsed(token)
309    }
310
311    /// Consumes the current token after performing applicable alias substitution.
312    ///
313    /// If the current token is not yet read from the underlying lexer, it is read.
314    ///
315    /// This function checks if the token is the name of an alias. If it is,
316    /// alias substitution is performed on the token and the result is
317    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
318    ///
319    /// Alias substitution is performed only if at least one of the following is
320    /// true:
321    ///
322    /// - The token is the first command word in a simple command, that is, it is
323    ///   the word for the command name. (This condition should be specified by the
324    ///   `is_command_name` parameter.)
325    /// - The token comes just after the replacement string of another alias
326    ///   substitution that ends with a blank character.
327    /// - The token names a global alias.
328    ///
329    /// However, alias substitution should _not_ be performed on a reserved word
330    /// in any case. It is your responsibility to check the token type and not to
331    /// call this function on a reserved word. That is why this function is named
332    /// `manual`. To consume a reserved word without performing alias
333    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
334    /// [`take_token_auto`](Self::take_token_auto).
335    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
336        let token = self.take_token_raw().await?;
337        Ok(self.substitute_alias(token, is_command_name))
338    }
339
340    /// Consumes the current token after performing applicable alias substitution.
341    ///
342    /// This function performs alias substitution unless the result is one of the
343    /// reserved words specified in the argument.
344    ///
345    /// Alias substitution is performed repeatedly until a non-alias token is
346    /// found. That is why this function is named `auto`. This function should be
347    /// used only in contexts where no backtrack is needed after alias
348    /// substitution. If you need to backtrack or want to know whether alias
349    /// substitution was performed or not, you should use
350    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
351    /// alias substitution at most once and returns `Rec`.
352    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
353        loop {
354            let token = self.take_token_raw().await?;
355            if let Token(Some(keyword)) = token.id {
356                if keywords.contains(&keyword) {
357                    return Ok(token);
358                }
359            }
360            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
361                return Ok(token);
362            }
363        }
364    }
365
366    /// Tests if there is a blank before the next token.
367    ///
368    /// This function can be called to tell whether the previous and next tokens
369    /// are separated by a blank or they are adjacent.
370    ///
371    /// This function must be called after the previous token has been taken (by
372    /// one of [`take_token_raw`](Self::take_token_raw),
373    /// [`take_token_manual`](Self::take_token_manual) and
374    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
375    /// [peeked](Self::peek_token). Otherwise, this function would panic.
376    ///
377    /// # Panics
378    ///
379    /// If the previous token has not been taken or the next token has been
380    /// peeked.
381    pub async fn has_blank(&mut self) -> Result<bool> {
382        assert!(self.token.is_none(), "There should be no pending token");
383        let c = self.lexer.peek_char().await?;
384        Ok(c.is_some_and(is_blank))
385    }
386
387    /// Remembers the given partial here-document for later parsing of its content.
388    ///
389    /// The remembered here-document's content will be parsed when
390    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
391    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
392        self.unread_here_docs.push(here_doc)
393    }
394
395    /// Reads here-document contents that matches the remembered list of
396    /// here-document operators.
397    ///
398    /// This function reads here-document contents corresponding to
399    /// here-document operators that have been saved with
400    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
401    /// The results are inserted to the `content` field of the `HereDoc`
402    /// instances.
403    ///
404    /// This function must be called just after a newline token has been taken
405    /// (either [manual](Self::take_token_manual) or
406    /// [auto](Self::take_token_auto)). If there is a pending token that has been
407    /// peeked but not yet taken, this function will panic!
408    pub async fn here_doc_contents(&mut self) -> Result<()> {
409        assert!(
410            self.token.is_none(),
411            "No token must be peeked before reading here-doc contents"
412        );
413
414        for here_doc in self.unread_here_docs.drain(..) {
415            self.lexer.here_doc_content(&here_doc).await?;
416        }
417
418        Ok(())
419    }
420
421    /// Ensures that there is no pending partial here-document.
422    ///
423    /// If there is any, this function returns a `MissingHereDocContent` error.
424    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
425        match self.unread_here_docs.first() {
426            None => Ok(()),
427            Some(here_doc) => Err(Error {
428                cause: SyntaxError::MissingHereDocContent.into(),
429                location: here_doc.delimiter.location.clone(),
430            }),
431        }
432    }
433
434    /// Determines whether a word names a declaration utility.
435    ///
436    /// See [`decl_utils`](crate::decl_util) for more information.
437    pub(super) fn word_names_declaration_utility(&self, word: &Word) -> Option<bool> {
438        if let Some(name) = word.to_string_if_literal() {
439            self.decl_utils.is_declaration_utility(&name)
440        } else {
441            Some(false)
442        }
443    }
444}
445
446#[allow(
447    clippy::bool_assert_comparison,
448    reason = "to make the expected values clearer"
449)]
450#[cfg(test)]
451mod tests {
452    use super::*;
453    use crate::alias::AliasSet;
454    use crate::alias::HashEntry;
455    use crate::source::Location;
456    use assert_matches::assert_matches;
457    use futures_util::FutureExt as _;
458    use std::cell::OnceCell;
459
460    #[test]
461    fn parser_take_token_manual_successful_substitution() {
462        let mut lexer = Lexer::with_code("X");
463        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
464        let mut aliases = AliasSet::new();
465        aliases.insert(HashEntry::new(
466            "X".to_string(),
467            "x".to_string(),
468            false,
469            Location::dummy("?"),
470        ));
471        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
472
473        let result = parser.take_token_manual(true).now_or_never().unwrap();
474        assert_matches!(result, Ok(Rec::AliasSubstituted));
475
476        let result = parser.take_token_manual(true).now_or_never().unwrap();
477        let token = result.unwrap().unwrap();
478        assert_eq!(token.to_string(), "x");
479    }
480
481    #[test]
482    fn parser_take_token_manual_not_command_name() {
483        let mut lexer = Lexer::with_code("X");
484        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
485        let mut aliases = AliasSet::new();
486        aliases.insert(HashEntry::new(
487            "X".to_string(),
488            "x".to_string(),
489            false,
490            Location::dummy("?"),
491        ));
492        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
493
494        let result = parser.take_token_manual(false).now_or_never().unwrap();
495        let token = result.unwrap().unwrap();
496        assert_eq!(token.to_string(), "X");
497    }
498
499    #[test]
500    fn parser_take_token_manual_not_literal() {
501        let mut lexer = Lexer::with_code(r"\X");
502        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
503        let mut aliases = AliasSet::new();
504        aliases.insert(HashEntry::new(
505            "X".to_string(),
506            "x".to_string(),
507            false,
508            Location::dummy("?"),
509        ));
510        aliases.insert(HashEntry::new(
511            r"\X".to_string(),
512            "quoted".to_string(),
513            false,
514            Location::dummy("?"),
515        ));
516        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
517
518        let result = parser.take_token_manual(true).now_or_never().unwrap();
519        let token = result.unwrap().unwrap();
520        assert_eq!(token.to_string(), r"\X");
521    }
522
523    #[test]
524    fn parser_take_token_manual_operator() {
525        let mut lexer = Lexer::with_code(";");
526        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
527        let mut aliases = AliasSet::new();
528        aliases.insert(HashEntry::new(
529            ";".to_string(),
530            "x".to_string(),
531            false,
532            Location::dummy("?"),
533        ));
534        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
535
536        let result = parser.take_token_manual(true).now_or_never().unwrap();
537        let token = result.unwrap().unwrap();
538        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
539        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
540    }
541
542    #[test]
543    fn parser_take_token_manual_no_match() {
544        let mut lexer = Lexer::with_code("X");
545        let mut parser = Parser::new(&mut lexer);
546
547        let result = parser.take_token_manual(true).now_or_never().unwrap();
548        let token = result.unwrap().unwrap();
549        assert_eq!(token.to_string(), "X");
550    }
551
552    #[test]
553    fn parser_take_token_manual_recursive_substitution() {
554        let mut lexer = Lexer::with_code("X");
555        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
556        let mut aliases = AliasSet::new();
557        aliases.insert(HashEntry::new(
558            "X".to_string(),
559            "Y x".to_string(),
560            false,
561            Location::dummy("?"),
562        ));
563        aliases.insert(HashEntry::new(
564            "Y".to_string(),
565            "X y".to_string(),
566            false,
567            Location::dummy("?"),
568        ));
569        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
570
571        let result = parser.take_token_manual(true).now_or_never().unwrap();
572        assert_matches!(result, Ok(Rec::AliasSubstituted));
573
574        let result = parser.take_token_manual(true).now_or_never().unwrap();
575        assert_matches!(result, Ok(Rec::AliasSubstituted));
576
577        let result = parser.take_token_manual(true).now_or_never().unwrap();
578        let token = result.unwrap().unwrap();
579        assert_eq!(token.to_string(), "X");
580
581        let result = parser.take_token_manual(true).now_or_never().unwrap();
582        let token = result.unwrap().unwrap();
583        assert_eq!(token.to_string(), "y");
584
585        let rec = parser.take_token_manual(true).now_or_never().unwrap();
586        let token = rec.unwrap().unwrap();
587        assert_eq!(token.to_string(), "x");
588    }
589
590    #[test]
591    fn parser_take_token_manual_after_blank_ending_substitution() {
592        let mut lexer = Lexer::with_code("X\tY");
593        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
594        let mut aliases = AliasSet::new();
595        aliases.insert(HashEntry::new(
596            "X".to_string(),
597            " X ".to_string(),
598            false,
599            Location::dummy("?"),
600        ));
601        aliases.insert(HashEntry::new(
602            "Y".to_string(),
603            "y".to_string(),
604            false,
605            Location::dummy("?"),
606        ));
607        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
608
609        let result = parser.take_token_manual(true).now_or_never().unwrap();
610        assert_matches!(result, Ok(Rec::AliasSubstituted));
611
612        let result = parser.take_token_manual(true).now_or_never().unwrap();
613        let token = result.unwrap().unwrap();
614        assert_eq!(token.to_string(), "X");
615
616        let result = parser.take_token_manual(false).now_or_never().unwrap();
617        assert_matches!(result, Ok(Rec::AliasSubstituted));
618
619        let result = parser.take_token_manual(false).now_or_never().unwrap();
620        let token = result.unwrap().unwrap();
621        assert_eq!(token.to_string(), "y");
622    }
623
624    #[test]
625    fn parser_take_token_manual_not_after_blank_ending_substitution() {
626        let mut lexer = Lexer::with_code("X\tY");
627        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
628        let mut aliases = AliasSet::new();
629        aliases.insert(HashEntry::new(
630            "X".to_string(),
631            " X".to_string(),
632            false,
633            Location::dummy("?"),
634        ));
635        aliases.insert(HashEntry::new(
636            "Y".to_string(),
637            "y".to_string(),
638            false,
639            Location::dummy("?"),
640        ));
641        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
642
643        let result = parser.take_token_manual(true).now_or_never().unwrap();
644        assert_matches!(result, Ok(Rec::AliasSubstituted));
645
646        let result = parser.take_token_manual(true).now_or_never().unwrap();
647        let token = result.unwrap().unwrap();
648        assert_eq!(token.to_string(), "X");
649
650        let result = parser.take_token_manual(false).now_or_never().unwrap();
651        let token = result.unwrap().unwrap();
652        assert_eq!(token.to_string(), "Y");
653    }
654
655    #[test]
656    fn parser_take_token_manual_global() {
657        let mut lexer = Lexer::with_code("X");
658        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
659        let mut aliases = AliasSet::new();
660        aliases.insert(HashEntry::new(
661            "X".to_string(),
662            "x".to_string(),
663            true,
664            Location::dummy("?"),
665        ));
666        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
667
668        let result = parser.take_token_manual(false).now_or_never().unwrap();
669        assert_matches!(result, Ok(Rec::AliasSubstituted));
670
671        let result = parser.take_token_manual(false).now_or_never().unwrap();
672        let token = result.unwrap().unwrap();
673        assert_eq!(token.to_string(), "x");
674    }
675
676    #[test]
677    fn parser_take_token_auto_non_keyword() {
678        let mut lexer = Lexer::with_code("X");
679        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
680        let mut aliases = AliasSet::new();
681        aliases.insert(HashEntry::new(
682            "X".to_string(),
683            "x".to_string(),
684            true,
685            Location::dummy("?"),
686        ));
687        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
688
689        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
690        assert_eq!(token.to_string(), "x");
691    }
692
693    #[test]
694    fn parser_take_token_auto_keyword_matched() {
695        let mut lexer = Lexer::with_code("if");
696        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
697        let mut aliases = AliasSet::new();
698        aliases.insert(HashEntry::new(
699            "if".to_string(),
700            "x".to_string(),
701            true,
702            Location::dummy("?"),
703        ));
704        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
705
706        let token = parser
707            .take_token_auto(&[Keyword::If])
708            .now_or_never()
709            .unwrap()
710            .unwrap();
711        assert_eq!(token.to_string(), "if");
712    }
713
714    #[test]
715    fn parser_take_token_auto_keyword_unmatched() {
716        let mut lexer = Lexer::with_code("if");
717        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
718        let mut aliases = AliasSet::new();
719        aliases.insert(HashEntry::new(
720            "if".to_string(),
721            "x".to_string(),
722            true,
723            Location::dummy("?"),
724        ));
725        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
726
727        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
728        assert_eq!(token.to_string(), "x");
729    }
730
731    #[test]
732    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
733        let mut lexer = Lexer::with_code("X");
734        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
735        let mut aliases = AliasSet::new();
736        aliases.insert(HashEntry::new(
737            "X".to_string(),
738            "if".to_string(),
739            true,
740            Location::dummy("?"),
741        ));
742        aliases.insert(HashEntry::new(
743            "if".to_string(),
744            "x".to_string(),
745            true,
746            Location::dummy("?"),
747        ));
748        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
749
750        let token = parser
751            .take_token_auto(&[Keyword::If])
752            .now_or_never()
753            .unwrap()
754            .unwrap();
755        assert_eq!(token.to_string(), "if");
756    }
757
758    #[test]
759    fn parser_has_blank_true() {
760        let mut lexer = Lexer::with_code(" ");
761        let mut parser = Parser::new(&mut lexer);
762        let result = parser.has_blank().now_or_never().unwrap();
763        assert_eq!(result, Ok(true));
764    }
765
766    #[test]
767    fn parser_has_blank_false() {
768        let mut lexer = Lexer::with_code("(");
769        let mut parser = Parser::new(&mut lexer);
770        let result = parser.has_blank().now_or_never().unwrap();
771        assert_eq!(result, Ok(false));
772    }
773
774    #[test]
775    fn parser_has_blank_eof() {
776        let mut lexer = Lexer::with_code("");
777        let mut parser = Parser::new(&mut lexer);
778        let result = parser.has_blank().now_or_never().unwrap();
779        assert_eq!(result, Ok(false));
780    }
781
782    #[test]
783    fn parser_has_blank_true_with_line_continuations() {
784        let mut lexer = Lexer::with_code("\\\n\\\n ");
785        let mut parser = Parser::new(&mut lexer);
786        let result = parser.has_blank().now_or_never().unwrap();
787        assert_eq!(result, Ok(true));
788    }
789
790    #[test]
791    fn parser_has_blank_false_with_line_continuations() {
792        let mut lexer = Lexer::with_code("\\\n\\\n\\\n(");
793        let mut parser = Parser::new(&mut lexer);
794        let result = parser.has_blank().now_or_never().unwrap();
795        assert_eq!(result, Ok(false));
796    }
797
798    #[test]
799    #[should_panic(expected = "There should be no pending token")]
800    fn parser_has_blank_with_pending_token() {
801        let mut lexer = Lexer::with_code("foo");
802        let mut parser = Parser::new(&mut lexer);
803        parser.peek_token().now_or_never().unwrap().unwrap();
804        let _ = parser.has_blank().now_or_never().unwrap();
805    }
806
807    #[test]
808    fn parser_reading_no_here_doc_contents() {
809        let mut lexer = Lexer::with_code("X");
810        let mut parser = Parser::new(&mut lexer);
811        parser.here_doc_contents().now_or_never().unwrap().unwrap();
812
813        let location = lexer.location().now_or_never().unwrap().unwrap();
814        assert_eq!(location.code.start_line_number.get(), 1);
815        assert_eq!(location.range, 0..1);
816    }
817
818    #[test]
819    fn parser_reading_one_here_doc_content() {
820        let delimiter = "END".parse().unwrap();
821
822        let mut lexer = Lexer::with_code("END\nX");
823        let mut parser = Parser::new(&mut lexer);
824        let remove_tabs = false;
825        let here_doc = Rc::new(HereDoc {
826            delimiter,
827            remove_tabs,
828            content: OnceCell::new(),
829        });
830        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
831        parser.here_doc_contents().now_or_never().unwrap().unwrap();
832        assert_eq!(here_doc.delimiter.to_string(), "END");
833        assert_eq!(here_doc.remove_tabs, remove_tabs);
834        assert_eq!(here_doc.content.get().unwrap().0, []);
835
836        let location = lexer.location().now_or_never().unwrap().unwrap();
837        assert_eq!(location.code.start_line_number.get(), 1);
838        assert_eq!(location.range, 4..5);
839    }
840
841    #[test]
842    fn parser_reading_many_here_doc_contents() {
843        let delimiter1 = "ONE".parse().unwrap();
844        let delimiter2 = "TWO".parse().unwrap();
845        let delimiter3 = "THREE".parse().unwrap();
846
847        let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX");
848        let mut parser = Parser::new(&mut lexer);
849        let here_doc1 = Rc::new(HereDoc {
850            delimiter: delimiter1,
851            remove_tabs: false,
852            content: OnceCell::new(),
853        });
854        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
855        let here_doc2 = Rc::new(HereDoc {
856            delimiter: delimiter2,
857            remove_tabs: true,
858            content: OnceCell::new(),
859        });
860        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
861        let here_doc3 = Rc::new(HereDoc {
862            delimiter: delimiter3,
863            remove_tabs: false,
864            content: OnceCell::new(),
865        });
866        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
867        parser.here_doc_contents().now_or_never().unwrap().unwrap();
868        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
869        assert_eq!(here_doc1.remove_tabs, false);
870        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
871        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
872        assert_eq!(here_doc2.remove_tabs, true);
873        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
874        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
875        assert_eq!(here_doc3.remove_tabs, false);
876        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
877    }
878
879    #[test]
880    fn parser_reading_here_doc_contents_twice() {
881        let delimiter1 = "ONE".parse().unwrap();
882        let delimiter2 = "TWO".parse().unwrap();
883
884        let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n");
885        let mut parser = Parser::new(&mut lexer);
886        let here_doc1 = Rc::new(HereDoc {
887            delimiter: delimiter1,
888            remove_tabs: false,
889            content: OnceCell::new(),
890        });
891        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
892        parser.here_doc_contents().now_or_never().unwrap().unwrap();
893        let here_doc2 = Rc::new(HereDoc {
894            delimiter: delimiter2,
895            remove_tabs: true,
896            content: OnceCell::new(),
897        });
898        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
899        parser.here_doc_contents().now_or_never().unwrap().unwrap();
900        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
901        assert_eq!(here_doc1.remove_tabs, false);
902        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
903        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
904        assert_eq!(here_doc2.remove_tabs, true);
905        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
906    }
907
908    #[test]
909    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
910    fn parser_here_doc_contents_must_be_called_without_pending_token() {
911        let mut lexer = Lexer::with_code("X");
912        let mut parser = Parser::new(&mut lexer);
913        parser.peek_token().now_or_never().unwrap().unwrap();
914        parser.here_doc_contents().now_or_never().unwrap().unwrap();
915    }
916}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
