yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::syntax::HereDoc;
31use crate::syntax::MaybeLiteral as _;
32use crate::syntax::Word;
33use std::rc::Rc;
34
35/// Entire result of parsing
36pub type Result<T> = std::result::Result<T, Error>;
37
38/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
39/// parsing after alias substitution
40///
41/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
42///
43/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
44/// meaningful value and is returned from a parsing function that has performed alias substitution
45/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
46/// new source code produced by the substitution so that the syntax is correctly recognized in the
47/// new code.
48///
49/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
50/// recognized as an alias name during parse of a simple command, the simple command parser
51/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
52/// parser function, to recognize the `!` reserved word token as negation.
53///
54/// When a parser function successfully parses something, it returns the result in the `Parsed`
55/// variant. The caller then continues the remaining parse.
56#[derive(Copy, Clone, Debug, Eq, PartialEq)]
57pub enum Rec<T> {
58    /// Result of alias substitution
59    AliasSubstituted,
60    /// Successful parse result
61    Parsed(T),
62}
63
64impl<T> Rec<T> {
65    /// Tests if `self` is `AliasSubstituted`.
66    pub fn is_alias_substituted(&self) -> bool {
67        match self {
68            Rec::AliasSubstituted => true,
69            Rec::Parsed(_) => false,
70        }
71    }
72
73    /// Extracts the result of successful parsing.
74    ///
75    /// # Panics
76    ///
77    /// If `self` is `AliasSubstituted`.
78    pub fn unwrap(self) -> T {
79        match self {
80            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
81            Rec::Parsed(v) => v,
82        }
83    }
84
85    /// Transforms the result value in `self`.
86    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
87    where
88        F: FnOnce(T) -> Result<U>,
89    {
90        match self {
91            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
92            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
93        }
94    }
95}
96
97/// Set of parameters for constructing a [`Parser`]
98///
99/// `Config` is a builder for constructing a parser. A [new](Self::new)
100/// configuration starts with default settings. You can customize them by
101/// calling methods that can be chained. Finally, you can create a parser by
102/// providing the lexer to the [`input`](Self::input) method.
103///
104/// Note that this struct is for configuring a parser. To configure the
105/// [`Lexer`], use [`yash_env::parser::Config`].
106#[derive(Debug)]
107#[must_use = "Config must be used to create a parser"]
108pub struct Config<'a> {
109    /// Collection of aliases the parser applies to substitute command words
110    aliases: &'a dyn crate::alias::Glossary,
111
112    /// Glossary that determines whether a command name is a declaration utility
113    decl_utils: &'a dyn crate::decl_util::Glossary,
114}
115
116impl<'a> Config<'a> {
117    /// Creates a new configuration with default settings.
118    ///
119    /// You can also call [`Parser::config`] to create a new configuration.
120    pub fn new() -> Self {
121        Self {
122            aliases: &crate::alias::EmptyGlossary,
123            decl_utils: &crate::decl_util::PosixGlossary,
124        }
125    }
126
127    /// Sets the glossary of aliases.
128    ///
129    /// The parser uses the glossary to look up aliases and substitute command
130    /// words. The default glossary is [empty](crate::alias::EmptyGlossary).
131    #[inline]
132    pub fn aliases(&mut self, aliases: &'a dyn Glossary) -> &mut Self {
133        self.aliases = aliases;
134        self
135    }
136
137    /// Sets the glossary of declaration utilities.
138    ///
139    /// The parser uses the glossary to determine whether a command name is a
140    /// declaration utility. The default glossary is [`PosixGlossary`], which
141    /// recognizes the declaration utilities defined by POSIX. You can make
142    /// arbitrary command names declaration utilities by providing a custom
143    /// glossary. To meet the POSIX standard, the glossary's
144    /// [`is_declaration_utility`] method must return:
145    ///
146    /// - `Some(true)` for `export` and `readonly`
147    /// - `None` for `command`
148    ///
149    /// For detailed information on declaration utilities, see the
150    /// [`decl_utils`] module.
151    ///
152    /// [`decl_utils`]: crate::decl_util
153    /// [`PosixGlossary`]: crate::decl_util::PosixGlossary
154    /// [`is_declaration_utility`]: crate::decl_util::Glossary::is_declaration_utility
155    #[inline]
156    pub fn declaration_utilities(
157        &mut self,
158        decl_utils: &'a dyn crate::decl_util::Glossary,
159    ) -> &mut Self {
160        self.decl_utils = decl_utils;
161        self
162    }
163
164    /// Creates a parser with the given lexer.
165    pub fn input<'b>(&self, lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
166        Parser {
167            lexer,
168            aliases: self.aliases,
169            decl_utils: self.decl_utils,
170            token: None,
171            unread_here_docs: Vec::new(),
172        }
173    }
174}
175
176impl Default for Config<'_> {
177    fn default() -> Self {
178        Self::new()
179    }
180}
181
182/// The shell syntax parser
183///
184/// A parser manages a set of data used in syntax parsing. It keeps a reference
185/// to a [lexer](Lexer) that provides tokens to parse. It also has some
186/// parameters that can be set by a [configuration](Config) and affect the
187/// parsing process.
188///
189/// The [`new`](Self::new) function directly creates a parser with default
190/// settings. If you want to customize the settings, you can use the
191/// [`config`](Self::config) function to create a configuration and then create a
192/// parser with the configuration.
193///
194/// # Parsing here-documents
195///
196/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
197/// with empty content. The parser creates the `HereDoc` instance when it finds
198/// a here-document operator, but it has not read its content at that time. When
199/// finding a newline token, the parser reads the content and fills it into the
200/// `HereDoc` instance.
201///
202/// Unless you are interested in parsing a specific syntactic construct that is
203/// only part of source code, you will want to use a function that returns a
204/// complete result filled with proper here-document contents if any.
205/// Then the [`command_line`](Self::command_line) function is for you.
206/// See also the [module documentation](super).
207#[derive(Debug)]
208#[must_use = "Parser must be used to parse syntax"]
209pub struct Parser<'a, 'b> {
210    /// Lexer that provides tokens
211    lexer: &'a mut Lexer<'b>,
212
213    /// Collection of aliases the parser applies to substitute command words
214    aliases: &'a dyn crate::alias::Glossary,
215
216    /// Glossary that determines whether a command name is a declaration utility
217    decl_utils: &'a dyn crate::decl_util::Glossary,
218
219    /// Token to parse next
220    ///
221    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
222    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
223    token: Option<Result<Token>>,
224
225    /// Here-documents without contents
226    ///
227    /// The here-document is added to this list when the parser finds a
228    /// here-document operator. After consuming the next newline token, the
229    /// parser reads and fills the contents, then clears this list.
230    unread_here_docs: Vec<Rc<HereDoc>>,
231}
232
233impl<'a, 'b> Parser<'a, 'b> {
234    /// Creates a new configuration with default settings.
235    ///
236    /// This is a synonym for [`Config::new`]. Customize the settings by calling
237    /// methods of the returned configuration and then create a parser by calling
238    /// its [`input`](Config::input) method.
239    #[inline(always)]
240    pub fn config() -> Config<'a> {
241        Config::new()
242    }
243
244    /// Creates a new parser based on the given lexer.
245    ///
246    /// The parser uses the lexer to read tokens. All other settings are default.
247    /// To customize the settings, use the [`config`](Self::config) function.
248    pub fn new(lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
249        Self::config().input(lexer)
250    }
251
252    /// Reads a next token if the current token is `None`.
253    async fn require_token(&mut self) {
254        #[allow(clippy::question_mark, reason = "false positive")]
255        // TODO https://github.com/rust-lang/rust-clippy/issues/9518
256        if self.token.is_none() {
257            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
258                Err(e)
259            } else {
260                self.lexer.token().await
261            });
262        }
263    }
264
265    /// Returns a reference to the current token.
266    ///
267    /// If the current token is not yet read from the underlying lexer, it is read.
268    pub async fn peek_token(&mut self) -> Result<&Token> {
269        self.require_token().await;
270        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
271    }
272
273    /// Consumes the current token without performing alias substitution.
274    ///
275    /// If the current token is not yet read from the underlying lexer, it is read.
276    ///
277    /// This function does not perform alias substitution and therefore should be
278    /// used only in context where no alias substitution is expected. Otherwise,
279    /// you should use [`take_token_manual`](Self::take_token_manual) or
280    /// [`take_token_auto`](Self::take_token_auto) instead.
281    pub async fn take_token_raw(&mut self) -> Result<Token> {
282        self.require_token().await;
283        self.token.take().unwrap()
284    }
285
286    /// Performs alias substitution on a token that has just been
287    /// [taken](Self::take_token_raw).
288    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
289        // TODO Only POSIXly-valid alias name should be recognized in POSIXly-correct mode.
290        if !self.aliases.is_empty()
291            && let Token(_) = token.id
292            && let Some(name) = token.word.to_string_if_literal()
293            && !token.word.location.code.source.is_alias_for(&name)
294            && let Some(alias) = self.aliases.look_up(&name)
295            && (is_command_name
296                || alias.global
297                || self.lexer.is_after_blank_ending_alias(token.index))
298        {
299            self.lexer.substitute_alias(token.index, &alias);
300            return Rec::AliasSubstituted;
301        }
302
303        Rec::Parsed(token)
304    }
305
306    /// Consumes the current token after performing applicable alias substitution.
307    ///
308    /// If the current token is not yet read from the underlying lexer, it is read.
309    ///
310    /// This function checks if the token is the name of an alias. If it is,
311    /// alias substitution is performed on the token and the result is
312    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
313    ///
314    /// Alias substitution is performed only if at least one of the following is
315    /// true:
316    ///
317    /// - The token is the first command word in a simple command, that is, it is
318    ///   the word for the command name. (This condition should be specified by the
319    ///   `is_command_name` parameter.)
320    /// - The token comes just after the replacement string of another alias
321    ///   substitution that ends with a blank character.
322    /// - The token names a global alias.
323    ///
324    /// However, alias substitution should _not_ be performed on a reserved word
325    /// in any case. It is your responsibility to check the token type and not to
326    /// call this function on a reserved word. That is why this function is named
327    /// `manual`. To consume a reserved word without performing alias
328    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
329    /// [`take_token_auto`](Self::take_token_auto).
330    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
331        let token = self.take_token_raw().await?;
332        Ok(self.substitute_alias(token, is_command_name))
333    }
334
335    /// Consumes the current token after performing applicable alias substitution.
336    ///
337    /// This function performs alias substitution unless the result is one of the
338    /// reserved words specified in the argument.
339    ///
340    /// Alias substitution is performed repeatedly until a non-alias token is
341    /// found. That is why this function is named `auto`. This function should be
342    /// used only in contexts where no backtrack is needed after alias
343    /// substitution. If you need to backtrack or want to know whether alias
344    /// substitution was performed or not, you should use
345    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
346    /// alias substitution at most once and returns `Rec`.
347    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
348        loop {
349            let token = self.take_token_raw().await?;
350            if let Token(Some(keyword)) = token.id
351                && keywords.contains(&keyword)
352            {
353                return Ok(token);
354            }
355            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
356                return Ok(token);
357            }
358        }
359    }
360
361    /// Tests if there is a blank before the next token.
362    ///
363    /// This function can be called to tell whether the previous and next tokens
364    /// are separated by a blank or they are adjacent.
365    ///
366    /// This function must be called after the previous token has been taken (by
367    /// one of [`take_token_raw`](Self::take_token_raw),
368    /// [`take_token_manual`](Self::take_token_manual) and
369    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
370    /// [peeked](Self::peek_token). Otherwise, this function would panic.
371    ///
372    /// # Panics
373    ///
374    /// If the previous token has not been taken or the next token has been
375    /// peeked.
376    pub async fn has_blank(&mut self) -> Result<bool> {
377        assert!(self.token.is_none(), "There should be no pending token");
378        let c = self.lexer.peek_char().await?;
379        Ok(c.is_some_and(is_blank))
380    }
381
382    /// Remembers the given partial here-document for later parsing of its content.
383    ///
384    /// The remembered here-document's content will be parsed when
385    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
386    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
387        self.unread_here_docs.push(here_doc)
388    }
389
390    /// Reads here-document contents that matches the remembered list of
391    /// here-document operators.
392    ///
393    /// This function reads here-document contents corresponding to
394    /// here-document operators that have been saved with
395    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
396    /// The results are inserted to the `content` field of the `HereDoc`
397    /// instances.
398    ///
399    /// This function must be called just after a newline token has been taken
400    /// (either [manual](Self::take_token_manual) or
401    /// [auto](Self::take_token_auto)). If there is a pending token that has been
402    /// peeked but not yet taken, this function will panic!
403    pub async fn here_doc_contents(&mut self) -> Result<()> {
404        assert!(
405            self.token.is_none(),
406            "No token must be peeked before reading here-doc contents"
407        );
408
409        for here_doc in self.unread_here_docs.drain(..) {
410            self.lexer.here_doc_content(&here_doc).await?;
411        }
412
413        Ok(())
414    }
415
416    /// Ensures that there is no pending partial here-document.
417    ///
418    /// If there is any, this function returns a `MissingHereDocContent` error.
419    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
420        match self.unread_here_docs.first() {
421            None => Ok(()),
422            Some(here_doc) => Err(Error {
423                cause: SyntaxError::MissingHereDocContent.into(),
424                location: here_doc.delimiter.location.clone(),
425            }),
426        }
427    }
428
429    /// Determines whether a word names a declaration utility.
430    ///
431    /// See [`decl_utils`](crate::decl_util) for more information.
432    pub(super) fn word_names_declaration_utility(&self, word: &Word) -> Option<bool> {
433        if let Some(name) = word.to_string_if_literal() {
434            self.decl_utils.is_declaration_utility(&name)
435        } else {
436            Some(false)
437        }
438    }
439}
440
441#[allow(
442    clippy::bool_assert_comparison,
443    reason = "to make the expected values clearer"
444)]
445#[cfg(test)]
446mod tests {
447    use super::*;
448    use crate::alias::AliasSet;
449    use crate::alias::HashEntry;
450    use crate::source::Location;
451    use futures_util::FutureExt as _;
452    use std::assert_matches;
453    use std::cell::OnceCell;
454
455    #[test]
456    fn parser_take_token_manual_successful_substitution() {
457        let mut lexer = Lexer::with_code("X");
458        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
459        let mut aliases = AliasSet::new();
460        aliases.insert(HashEntry::new(
461            "X".to_string(),
462            "x".to_string(),
463            false,
464            Location::dummy("?"),
465        ));
466        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
467
468        let result = parser.take_token_manual(true).now_or_never().unwrap();
469        assert_matches!(result, Ok(Rec::AliasSubstituted));
470
471        let result = parser.take_token_manual(true).now_or_never().unwrap();
472        let token = result.unwrap().unwrap();
473        assert_eq!(token.to_string(), "x");
474    }
475
476    #[test]
477    fn parser_take_token_manual_not_command_name() {
478        let mut lexer = Lexer::with_code("X");
479        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
480        let mut aliases = AliasSet::new();
481        aliases.insert(HashEntry::new(
482            "X".to_string(),
483            "x".to_string(),
484            false,
485            Location::dummy("?"),
486        ));
487        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
488
489        let result = parser.take_token_manual(false).now_or_never().unwrap();
490        let token = result.unwrap().unwrap();
491        assert_eq!(token.to_string(), "X");
492    }
493
494    #[test]
495    fn parser_take_token_manual_not_literal() {
496        let mut lexer = Lexer::with_code(r"\X");
497        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
498        let mut aliases = AliasSet::new();
499        aliases.insert(HashEntry::new(
500            "X".to_string(),
501            "x".to_string(),
502            false,
503            Location::dummy("?"),
504        ));
505        aliases.insert(HashEntry::new(
506            r"\X".to_string(),
507            "quoted".to_string(),
508            false,
509            Location::dummy("?"),
510        ));
511        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
512
513        let result = parser.take_token_manual(true).now_or_never().unwrap();
514        let token = result.unwrap().unwrap();
515        assert_eq!(token.to_string(), r"\X");
516    }
517
518    #[test]
519    fn parser_take_token_manual_operator() {
520        let mut lexer = Lexer::with_code(";");
521        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
522        let mut aliases = AliasSet::new();
523        aliases.insert(HashEntry::new(
524            ";".to_string(),
525            "x".to_string(),
526            false,
527            Location::dummy("?"),
528        ));
529        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
530
531        let result = parser.take_token_manual(true).now_or_never().unwrap();
532        let token = result.unwrap().unwrap();
533        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
534        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
535    }
536
537    #[test]
538    fn parser_take_token_manual_no_match() {
539        let mut lexer = Lexer::with_code("X");
540        let mut parser = Parser::new(&mut lexer);
541
542        let result = parser.take_token_manual(true).now_or_never().unwrap();
543        let token = result.unwrap().unwrap();
544        assert_eq!(token.to_string(), "X");
545    }
546
547    #[test]
548    fn parser_take_token_manual_recursive_substitution() {
549        let mut lexer = Lexer::with_code("X");
550        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
551        let mut aliases = AliasSet::new();
552        aliases.insert(HashEntry::new(
553            "X".to_string(),
554            "Y x".to_string(),
555            false,
556            Location::dummy("?"),
557        ));
558        aliases.insert(HashEntry::new(
559            "Y".to_string(),
560            "X y".to_string(),
561            false,
562            Location::dummy("?"),
563        ));
564        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
565
566        let result = parser.take_token_manual(true).now_or_never().unwrap();
567        assert_matches!(result, Ok(Rec::AliasSubstituted));
568
569        let result = parser.take_token_manual(true).now_or_never().unwrap();
570        assert_matches!(result, Ok(Rec::AliasSubstituted));
571
572        let result = parser.take_token_manual(true).now_or_never().unwrap();
573        let token = result.unwrap().unwrap();
574        assert_eq!(token.to_string(), "X");
575
576        let result = parser.take_token_manual(true).now_or_never().unwrap();
577        let token = result.unwrap().unwrap();
578        assert_eq!(token.to_string(), "y");
579
580        let rec = parser.take_token_manual(true).now_or_never().unwrap();
581        let token = rec.unwrap().unwrap();
582        assert_eq!(token.to_string(), "x");
583    }
584
585    #[test]
586    fn parser_take_token_manual_after_blank_ending_substitution() {
587        let mut lexer = Lexer::with_code("X\tY");
588        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
589        let mut aliases = AliasSet::new();
590        aliases.insert(HashEntry::new(
591            "X".to_string(),
592            " X ".to_string(),
593            false,
594            Location::dummy("?"),
595        ));
596        aliases.insert(HashEntry::new(
597            "Y".to_string(),
598            "y".to_string(),
599            false,
600            Location::dummy("?"),
601        ));
602        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
603
604        let result = parser.take_token_manual(true).now_or_never().unwrap();
605        assert_matches!(result, Ok(Rec::AliasSubstituted));
606
607        let result = parser.take_token_manual(true).now_or_never().unwrap();
608        let token = result.unwrap().unwrap();
609        assert_eq!(token.to_string(), "X");
610
611        let result = parser.take_token_manual(false).now_or_never().unwrap();
612        assert_matches!(result, Ok(Rec::AliasSubstituted));
613
614        let result = parser.take_token_manual(false).now_or_never().unwrap();
615        let token = result.unwrap().unwrap();
616        assert_eq!(token.to_string(), "y");
617    }
618
619    #[test]
620    fn parser_take_token_manual_not_after_blank_ending_substitution() {
621        let mut lexer = Lexer::with_code("X\tY");
622        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
623        let mut aliases = AliasSet::new();
624        aliases.insert(HashEntry::new(
625            "X".to_string(),
626            " X".to_string(),
627            false,
628            Location::dummy("?"),
629        ));
630        aliases.insert(HashEntry::new(
631            "Y".to_string(),
632            "y".to_string(),
633            false,
634            Location::dummy("?"),
635        ));
636        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
637
638        let result = parser.take_token_manual(true).now_or_never().unwrap();
639        assert_matches!(result, Ok(Rec::AliasSubstituted));
640
641        let result = parser.take_token_manual(true).now_or_never().unwrap();
642        let token = result.unwrap().unwrap();
643        assert_eq!(token.to_string(), "X");
644
645        let result = parser.take_token_manual(false).now_or_never().unwrap();
646        let token = result.unwrap().unwrap();
647        assert_eq!(token.to_string(), "Y");
648    }
649
650    #[test]
651    fn parser_take_token_manual_global() {
652        let mut lexer = Lexer::with_code("X");
653        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
654        let mut aliases = AliasSet::new();
655        aliases.insert(HashEntry::new(
656            "X".to_string(),
657            "x".to_string(),
658            true,
659            Location::dummy("?"),
660        ));
661        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
662
663        let result = parser.take_token_manual(false).now_or_never().unwrap();
664        assert_matches!(result, Ok(Rec::AliasSubstituted));
665
666        let result = parser.take_token_manual(false).now_or_never().unwrap();
667        let token = result.unwrap().unwrap();
668        assert_eq!(token.to_string(), "x");
669    }
670
671    #[test]
672    fn parser_take_token_auto_non_keyword() {
673        let mut lexer = Lexer::with_code("X");
674        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
675        let mut aliases = AliasSet::new();
676        aliases.insert(HashEntry::new(
677            "X".to_string(),
678            "x".to_string(),
679            true,
680            Location::dummy("?"),
681        ));
682        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
683
684        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
685        assert_eq!(token.to_string(), "x");
686    }
687
688    #[test]
689    fn parser_take_token_auto_keyword_matched() {
690        let mut lexer = Lexer::with_code("if");
691        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
692        let mut aliases = AliasSet::new();
693        aliases.insert(HashEntry::new(
694            "if".to_string(),
695            "x".to_string(),
696            true,
697            Location::dummy("?"),
698        ));
699        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
700
701        let token = parser
702            .take_token_auto(&[Keyword::If])
703            .now_or_never()
704            .unwrap()
705            .unwrap();
706        assert_eq!(token.to_string(), "if");
707    }
708
709    #[test]
710    fn parser_take_token_auto_keyword_unmatched() {
711        let mut lexer = Lexer::with_code("if");
712        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
713        let mut aliases = AliasSet::new();
714        aliases.insert(HashEntry::new(
715            "if".to_string(),
716            "x".to_string(),
717            true,
718            Location::dummy("?"),
719        ));
720        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
721
722        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
723        assert_eq!(token.to_string(), "x");
724    }
725
726    #[test]
727    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
728        let mut lexer = Lexer::with_code("X");
729        #[allow(clippy::mutable_key_type, reason = "AliasSet is defined as such")]
730        let mut aliases = AliasSet::new();
731        aliases.insert(HashEntry::new(
732            "X".to_string(),
733            "if".to_string(),
734            true,
735            Location::dummy("?"),
736        ));
737        aliases.insert(HashEntry::new(
738            "if".to_string(),
739            "x".to_string(),
740            true,
741            Location::dummy("?"),
742        ));
743        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
744
745        let token = parser
746            .take_token_auto(&[Keyword::If])
747            .now_or_never()
748            .unwrap()
749            .unwrap();
750        assert_eq!(token.to_string(), "if");
751    }
752
753    #[test]
754    fn parser_has_blank_true() {
755        let mut lexer = Lexer::with_code(" ");
756        let mut parser = Parser::new(&mut lexer);
757        let result = parser.has_blank().now_or_never().unwrap();
758        assert_eq!(result, Ok(true));
759    }
760
761    #[test]
762    fn parser_has_blank_false() {
763        let mut lexer = Lexer::with_code("(");
764        let mut parser = Parser::new(&mut lexer);
765        let result = parser.has_blank().now_or_never().unwrap();
766        assert_eq!(result, Ok(false));
767    }
768
769    #[test]
770    fn parser_has_blank_eof() {
771        let mut lexer = Lexer::with_code("");
772        let mut parser = Parser::new(&mut lexer);
773        let result = parser.has_blank().now_or_never().unwrap();
774        assert_eq!(result, Ok(false));
775    }
776
777    #[test]
778    fn parser_has_blank_true_with_line_continuations() {
779        let mut lexer = Lexer::with_code("\\\n\\\n ");
780        let mut parser = Parser::new(&mut lexer);
781        let result = parser.has_blank().now_or_never().unwrap();
782        assert_eq!(result, Ok(true));
783    }
784
785    #[test]
786    fn parser_has_blank_false_with_line_continuations() {
787        let mut lexer = Lexer::with_code("\\\n\\\n\\\n(");
788        let mut parser = Parser::new(&mut lexer);
789        let result = parser.has_blank().now_or_never().unwrap();
790        assert_eq!(result, Ok(false));
791    }
792
793    #[test]
794    #[should_panic(expected = "There should be no pending token")]
795    fn parser_has_blank_with_pending_token() {
796        let mut lexer = Lexer::with_code("foo");
797        let mut parser = Parser::new(&mut lexer);
798        parser.peek_token().now_or_never().unwrap().unwrap();
799        let _ = parser.has_blank().now_or_never().unwrap();
800    }
801
802    #[test]
803    fn parser_reading_no_here_doc_contents() {
804        let mut lexer = Lexer::with_code("X");
805        let mut parser = Parser::new(&mut lexer);
806        parser.here_doc_contents().now_or_never().unwrap().unwrap();
807
808        let location = lexer.location().now_or_never().unwrap().unwrap();
809        assert_eq!(location.code.start_line_number.get(), 1);
810        assert_eq!(location.range, 0..1);
811    }
812
813    #[test]
814    fn parser_reading_one_here_doc_content() {
815        let delimiter = "END".parse().unwrap();
816
817        let mut lexer = Lexer::with_code("END\nX");
818        let mut parser = Parser::new(&mut lexer);
819        let remove_tabs = false;
820        let here_doc = Rc::new(HereDoc {
821            delimiter,
822            remove_tabs,
823            content: OnceCell::new(),
824        });
825        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
826        parser.here_doc_contents().now_or_never().unwrap().unwrap();
827        assert_eq!(here_doc.delimiter.to_string(), "END");
828        assert_eq!(here_doc.remove_tabs, remove_tabs);
829        assert_eq!(here_doc.content.get().unwrap().0, []);
830
831        let location = lexer.location().now_or_never().unwrap().unwrap();
832        assert_eq!(location.code.start_line_number.get(), 1);
833        assert_eq!(location.range, 4..5);
834    }
835
836    #[test]
837    fn parser_reading_many_here_doc_contents() {
838        let delimiter1 = "ONE".parse().unwrap();
839        let delimiter2 = "TWO".parse().unwrap();
840        let delimiter3 = "THREE".parse().unwrap();
841
842        let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX");
843        let mut parser = Parser::new(&mut lexer);
844        let here_doc1 = Rc::new(HereDoc {
845            delimiter: delimiter1,
846            remove_tabs: false,
847            content: OnceCell::new(),
848        });
849        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
850        let here_doc2 = Rc::new(HereDoc {
851            delimiter: delimiter2,
852            remove_tabs: true,
853            content: OnceCell::new(),
854        });
855        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
856        let here_doc3 = Rc::new(HereDoc {
857            delimiter: delimiter3,
858            remove_tabs: false,
859            content: OnceCell::new(),
860        });
861        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
862        parser.here_doc_contents().now_or_never().unwrap().unwrap();
863        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
864        assert_eq!(here_doc1.remove_tabs, false);
865        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
866        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
867        assert_eq!(here_doc2.remove_tabs, true);
868        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
869        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
870        assert_eq!(here_doc3.remove_tabs, false);
871        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
872    }
873
874    #[test]
875    fn parser_reading_here_doc_contents_twice() {
876        let delimiter1 = "ONE".parse().unwrap();
877        let delimiter2 = "TWO".parse().unwrap();
878
879        let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n");
880        let mut parser = Parser::new(&mut lexer);
881        let here_doc1 = Rc::new(HereDoc {
882            delimiter: delimiter1,
883            remove_tabs: false,
884            content: OnceCell::new(),
885        });
886        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
887        parser.here_doc_contents().now_or_never().unwrap().unwrap();
888        let here_doc2 = Rc::new(HereDoc {
889            delimiter: delimiter2,
890            remove_tabs: true,
891            content: OnceCell::new(),
892        });
893        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
894        parser.here_doc_contents().now_or_never().unwrap().unwrap();
895        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
896        assert_eq!(here_doc1.remove_tabs, false);
897        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
898        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
899        assert_eq!(here_doc2.remove_tabs, true);
900        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
901    }
902
903    #[test]
904    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
905    fn parser_here_doc_contents_must_be_called_without_pending_token() {
906        let mut lexer = Lexer::with_code("X");
907        let mut parser = Parser::new(&mut lexer);
908        parser.peek_token().now_or_never().unwrap().unwrap();
909        parser.here_doc_contents().now_or_never().unwrap().unwrap();
910    }
911}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
