yash_syntax/parser/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamentals for implementing the parser
18//!
19//! This module includes common types that are used as building blocks for constructing the syntax
20//! parser.
21
22use super::error::Error;
23use super::error::SyntaxError;
24use super::lex::Keyword;
25use super::lex::Lexer;
26use super::lex::Token;
27use super::lex::TokenId::*;
28use crate::alias::Glossary;
29use crate::parser::lex::is_blank;
30use crate::syntax::HereDoc;
31use crate::syntax::MaybeLiteral;
32use crate::syntax::Word;
33use std::rc::Rc;
34
35/// Entire result of parsing
36pub type Result<T> = std::result::Result<T, Error>;
37
38/// Modifier that makes a result of parsing optional in order to trigger the parser to restart
39/// parsing after alias substitution
40///
41/// `Rec` stands for "recursion", as it is used to make the parser work recursively.
42///
43/// This enum type has two variants: `AliasSubstituted` and `Parsed`. The former contains no
44/// meaningful value and is returned from a parsing function that has performed alias substitution
45/// without consuming any tokens. In this case, the caller of the parsing function must inspect the
46/// new source code produced by the substitution so that the syntax is correctly recognized in the
47/// new code.
48///
49/// Assume we have an alias definition `untrue='! true'`, for example. When the word `untrue` is
50/// recognized as an alias name during parse of a simple command, the simple command parser
51/// function must stop parsing and return `AliasSubstituted`. This allows the caller, the pipeline
52/// parser function, to recognize the `!` reserved word token as negation.
53///
54/// When a parser function successfully parses something, it returns the result in the `Parsed`
55/// variant. The caller then continues the remaining parse.
56#[derive(Copy, Clone, Debug, Eq, PartialEq)]
57pub enum Rec<T> {
58    /// Result of alias substitution
59    AliasSubstituted,
60    /// Successful parse result
61    Parsed(T),
62}
63
64impl<T> Rec<T> {
65    /// Tests if `self` is `AliasSubstituted`.
66    pub fn is_alias_substituted(&self) -> bool {
67        match self {
68            Rec::AliasSubstituted => true,
69            Rec::Parsed(_) => false,
70        }
71    }
72
73    /// Extracts the result of successful parsing.
74    ///
75    /// # Panics
76    ///
77    /// If `self` is `AliasSubstituted`.
78    pub fn unwrap(self) -> T {
79        match self {
80            Rec::AliasSubstituted => panic!("Rec::AliasSubstituted cannot be unwrapped"),
81            Rec::Parsed(v) => v,
82        }
83    }
84
85    /// Transforms the result value in `self`.
86    pub fn map<U, F>(self, f: F) -> Result<Rec<U>>
87    where
88        F: FnOnce(T) -> Result<U>,
89    {
90        match self {
91            Rec::AliasSubstituted => Ok(Rec::AliasSubstituted),
92            Rec::Parsed(t) => Ok(Rec::Parsed(f(t)?)),
93        }
94    }
95}
96
97/// Set of parameters for constructing a [parser](Parser)
98///
99/// `Config` is a builder for constructing a parser. A [new](Self::new)
100/// configuration starts with default settings. You can customize them by
101/// calling methods that can be chained. Finally, you can create a parser by
102/// providing the lexer to the [`input`](Self::input) method.
103#[derive(Debug)]
104#[must_use = "Config must be used to create a parser"]
105pub struct Config<'a> {
106    /// Collection of aliases the parser applies to substitute command words
107    aliases: &'a dyn crate::alias::Glossary,
108
109    /// Glossary that determines whether a command name is a declaration utility
110    decl_utils: &'a dyn crate::decl_util::Glossary,
111}
112
113impl<'a> Config<'a> {
114    /// Creates a new configuration with default settings.
115    ///
116    /// You can also call [`Parser::config`] to create a new configuration.
117    pub fn new() -> Self {
118        Self {
119            aliases: &crate::alias::EmptyGlossary,
120            decl_utils: &crate::decl_util::PosixGlossary,
121        }
122    }
123
124    /// Sets the glossary of aliases.
125    ///
126    /// The parser uses the glossary to look up aliases and substitute command
127    /// words. The default glossary is [empty](crate::alias::EmptyGlossary).
128    #[inline]
129    pub fn aliases(&mut self, aliases: &'a dyn Glossary) -> &mut Self {
130        self.aliases = aliases;
131        self
132    }
133
134    /// Sets the glossary of declaration utilities.
135    ///
136    /// The parser uses the glossary to determine whether a command name is a
137    /// declaration utility. The default glossary is [`PosixGlossary`], which
138    /// recognizes the declaration utilities defined by POSIX. You can make
139    /// arbitrary command names declaration utilities by providing a custom
140    /// glossary. To meet the POSIX standard, the glossary's
141    /// [`is_declaration_utility`] method must return:
142    ///
143    /// - `Some(true)` for `export` and `readonly`
144    /// - `None` for `command`
145    ///
146    /// For detailed information on declaration utilities, see the
147    /// [`decl_utils`] module.
148    ///
149    /// [`decl_utils`]: crate::decl_util
150    /// [`PosixGlossary`]: crate::decl_util::PosixGlossary
151    /// [`is_declaration_utility`]: crate::decl_util::Glossary::is_declaration_utility
152    #[inline]
153    pub fn declaration_utilities(
154        &mut self,
155        decl_utils: &'a dyn crate::decl_util::Glossary,
156    ) -> &mut Self {
157        self.decl_utils = decl_utils;
158        self
159    }
160
161    /// Creates a parser with the given lexer.
162    pub fn input<'b>(&self, lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
163        Parser {
164            lexer,
165            aliases: self.aliases,
166            decl_utils: self.decl_utils,
167            token: None,
168            unread_here_docs: Vec::new(),
169        }
170    }
171}
172
173impl Default for Config<'_> {
174    fn default() -> Self {
175        Self::new()
176    }
177}
178
179/// The shell syntax parser
180///
181/// A parser manages a set of data used in syntax parsing. It keeps a reference
182/// to a [lexer](Lexer) that provides tokens to parse. It also has some
183/// parameters that can be set by a [configuration](Config) and affect the
184/// parsing process.
185///
186/// The [`new`](Self::new) function directly creates a parser with default
187/// settings. If you want to customize the settings, you can use the
188/// [`config`](Self::config) function to create a configuration and then create a
189/// parser with the configuration.
190///
191/// # Parsing here-documents
192///
193/// Most intrinsic functions of `Parser` may return an AST containing `HereDoc`s
194/// with empty content. The parser creates the `HereDoc` instance when it finds
195/// a here-document operator, but it has not read its content at that time. When
196/// finding a newline token, the parser reads the content and fills it into the
197/// `HereDoc` instance.
198///
199/// Unless you are interested in parsing a specific syntactic construct that is
200/// only part of source code, you will want to use a function that returns a
201/// complete result filled with proper here-document contents if any.
202/// Then the [`command_line`](Self::command_line) function is for you.
203/// See also the [module documentation](super).
204#[derive(Debug)]
205#[must_use = "Parser must be used to parse syntax"]
206pub struct Parser<'a, 'b> {
207    /// Lexer that provides tokens
208    lexer: &'a mut Lexer<'b>,
209
210    /// Collection of aliases the parser applies to substitute command words
211    aliases: &'a dyn crate::alias::Glossary,
212
213    /// Glossary that determines whether a command name is a declaration utility
214    decl_utils: &'a dyn crate::decl_util::Glossary,
215
216    /// Token to parse next
217    ///
218    /// This value is an option of a result. It is `None` when the next token is not yet parsed by
219    /// the lexer. It is `Some(Err(_))` if the lexer has failed.
220    token: Option<Result<Token>>,
221
222    /// Here-documents without contents
223    ///
224    /// The here-document is added to this list when the parser finds a
225    /// here-document operator. After consuming the next newline token, the
226    /// parser reads and fills the contents, then clears this list.
227    unread_here_docs: Vec<Rc<HereDoc>>,
228}
229
230impl<'a, 'b> Parser<'a, 'b> {
231    /// Creates a new configuration with default settings.
232    ///
233    /// This is a synonym for [`Config::new`]. Customize the settings by calling
234    /// methods of the returned configuration and then create a parser by calling
235    /// its [`input`](Config::input) method.
236    #[inline(always)]
237    pub fn config() -> Config<'a> {
238        Config::new()
239    }
240
241    /// Creates a new parser based on the given lexer.
242    ///
243    /// The parser uses the lexer to read tokens. All other settings are default.
244    /// To customize the settings, use the [`config`](Self::config) function.
245    pub fn new(lexer: &'a mut Lexer<'b>) -> Parser<'a, 'b> {
246        Self::config().input(lexer)
247    }
248
249    /// Reads a next token if the current token is `None`.
250    async fn require_token(&mut self) {
251        #[allow(clippy::question_mark)] // TODO https://github.com/rust-lang/rust-clippy/issues/9518
252        if self.token.is_none() {
253            self.token = Some(if let Err(e) = self.lexer.skip_blanks_and_comment().await {
254                Err(e)
255            } else {
256                self.lexer.token().await
257            });
258        }
259    }
260
261    /// Returns a reference to the current token.
262    ///
263    /// If the current token is not yet read from the underlying lexer, it is read.
264    pub async fn peek_token(&mut self) -> Result<&Token> {
265        self.require_token().await;
266        self.token.as_ref().unwrap().as_ref().map_err(|e| e.clone())
267    }
268
269    /// Consumes the current token without performing alias substitution.
270    ///
271    /// If the current token is not yet read from the underlying lexer, it is read.
272    ///
273    /// This function does not perform alias substitution and therefore should be
274    /// used only in context where no alias substitution is expected. Otherwise,
275    /// you should use [`take_token_manual`](Self::take_token_manual) or
276    /// [`take_token_auto`](Self::take_token_auto) instead.
277    pub async fn take_token_raw(&mut self) -> Result<Token> {
278        self.require_token().await;
279        self.token.take().unwrap()
280    }
281
282    /// Performs alias substitution on a token that has just been
283    /// [taken](Self::take_token_raw).
284    fn substitute_alias(&mut self, token: Token, is_command_name: bool) -> Rec<Token> {
285        // TODO Only POSIXly-valid alias name should be recognized in POSIXly-correct mode.
286        if !self.aliases.is_empty() {
287            if let Token(_) = token.id {
288                if let Some(name) = token.word.to_string_if_literal() {
289                    if !token.word.location.code.source.is_alias_for(&name) {
290                        if let Some(alias) = self.aliases.look_up(&name) {
291                            if is_command_name
292                                || alias.global
293                                || self.lexer.is_after_blank_ending_alias(token.index)
294                            {
295                                self.lexer.substitute_alias(token.index, &alias);
296                                return Rec::AliasSubstituted;
297                            }
298                        }
299                    }
300                }
301            }
302        }
303
304        Rec::Parsed(token)
305    }
306
307    /// Consumes the current token after performing applicable alias substitution.
308    ///
309    /// If the current token is not yet read from the underlying lexer, it is read.
310    ///
311    /// This function checks if the token is the name of an alias. If it is,
312    /// alias substitution is performed on the token and the result is
313    /// `Ok(AliasSubstituted)`. Otherwise, the token is consumed and returned.
314    ///
315    /// Alias substitution is performed only if at least one of the following is
316    /// true:
317    ///
318    /// - The token is the first command word in a simple command, that is, it is
319    ///   the word for the command name. (This condition should be specified by the
320    ///   `is_command_name` parameter.)
321    /// - The token comes just after the replacement string of another alias
322    ///   substitution that ends with a blank character.
323    /// - The token names a global alias.
324    ///
325    /// However, alias substitution should _not_ be performed on a reserved word
326    /// in any case. It is your responsibility to check the token type and not to
327    /// call this function on a reserved word. That is why this function is named
328    /// `manual`. To consume a reserved word without performing alias
329    /// substitution, you should call [`take_token_raw`](Self::take_token_raw) or
330    /// [`take_token_auto`](Self::take_token_auto).
331    pub async fn take_token_manual(&mut self, is_command_name: bool) -> Result<Rec<Token>> {
332        let token = self.take_token_raw().await?;
333        Ok(self.substitute_alias(token, is_command_name))
334    }
335
336    /// Consumes the current token after performing applicable alias substitution.
337    ///
338    /// This function performs alias substitution unless the result is one of the
339    /// reserved words specified in the argument.
340    ///
341    /// Alias substitution is performed repeatedly until a non-alias token is
342    /// found. That is why this function is named `auto`. This function should be
343    /// used only in contexts where no backtrack is needed after alias
344    /// substitution. If you need to backtrack or want to know whether alias
345    /// substitution was performed or not, you should use
346    /// [`Self::take_token_manual`](Self::take_token_manual), which performs
347    /// alias substitution at most once and returns `Rec`.
348    pub async fn take_token_auto(&mut self, keywords: &[Keyword]) -> Result<Token> {
349        loop {
350            let token = self.take_token_raw().await?;
351            if let Token(Some(keyword)) = token.id {
352                if keywords.contains(&keyword) {
353                    return Ok(token);
354                }
355            }
356            if let Rec::Parsed(token) = self.substitute_alias(token, false) {
357                return Ok(token);
358            }
359        }
360    }
361
362    /// Tests if there is a blank before the next token.
363    ///
364    /// This function can be called to tell whether the previous and next tokens
365    /// are separated by a blank or they are adjacent.
366    ///
367    /// This function must be called after the previous token has been taken (by
368    /// one of [`take_token_raw`](Self::take_token_raw),
369    /// [`take_token_manual`](Self::take_token_manual) and
370    /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
371    /// [peeked](Self::peek_token). Otherwise, this function would panic.
372    ///
373    /// # Panics
374    ///
375    /// If the previous token has not been taken or the next token has been
376    /// peeked.
377    pub async fn has_blank(&mut self) -> Result<bool> {
378        assert!(self.token.is_none(), "There should be no pending token");
379        let c = self.lexer.peek_char().await?;
380        Ok(c.is_some_and(is_blank))
381    }
382
383    /// Remembers the given partial here-document for later parsing of its content.
384    ///
385    /// The remembered here-document's content will be parsed when
386    /// [`here_doc_contents`](Self::here_doc_contents) is called later.
387    pub fn memorize_unread_here_doc(&mut self, here_doc: Rc<HereDoc>) {
388        self.unread_here_docs.push(here_doc)
389    }
390
391    /// Reads here-document contents that matches the remembered list of
392    /// here-document operators.
393    ///
394    /// This function reads here-document contents corresponding to
395    /// here-document operators that have been saved with
396    /// [`memorize_unread_here_doc`](Self::memorize_unread_here_doc).
397    /// The results are inserted to the `content` field of the `HereDoc`
398    /// instances.
399    ///
400    /// This function must be called just after a newline token has been taken
401    /// (either [manual](Self::take_token_manual) or
402    /// [auto](Self::take_token_auto)). If there is a pending token that has been
403    /// peeked but not yet taken, this function will panic!
404    pub async fn here_doc_contents(&mut self) -> Result<()> {
405        assert!(
406            self.token.is_none(),
407            "No token must be peeked before reading here-doc contents"
408        );
409
410        for here_doc in self.unread_here_docs.drain(..) {
411            self.lexer.here_doc_content(&here_doc).await?;
412        }
413
414        Ok(())
415    }
416
417    /// Ensures that there is no pending partial here-document.
418    ///
419    /// If there is any, this function returns a `MissingHereDocContent` error.
420    pub fn ensure_no_unread_here_doc(&self) -> Result<()> {
421        match self.unread_here_docs.first() {
422            None => Ok(()),
423            Some(here_doc) => Err(Error {
424                cause: SyntaxError::MissingHereDocContent.into(),
425                location: here_doc.delimiter.location.clone(),
426            }),
427        }
428    }
429
430    /// Determines whether a word names a declaration utility.
431    ///
432    /// See [`decl_utils`](crate::decl_util) for more information.
433    pub(super) fn word_names_declaration_utility(&self, word: &Word) -> Option<bool> {
434        if let Some(name) = word.to_string_if_literal() {
435            self.decl_utils.is_declaration_utility(&name)
436        } else {
437            Some(false)
438        }
439    }
440}
441
442#[allow(clippy::bool_assert_comparison)]
443#[cfg(test)]
444mod tests {
445    use super::*;
446    use crate::alias::AliasSet;
447    use crate::alias::HashEntry;
448    use crate::source::Location;
449    use assert_matches::assert_matches;
450    use futures_util::FutureExt;
451    use std::cell::OnceCell;
452
453    #[test]
454    fn parser_take_token_manual_successful_substitution() {
455        let mut lexer = Lexer::with_code("X");
456        #[allow(clippy::mutable_key_type)]
457        let mut aliases = AliasSet::new();
458        aliases.insert(HashEntry::new(
459            "X".to_string(),
460            "x".to_string(),
461            false,
462            Location::dummy("?"),
463        ));
464        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
465
466        let result = parser.take_token_manual(true).now_or_never().unwrap();
467        assert_matches!(result, Ok(Rec::AliasSubstituted));
468
469        let result = parser.take_token_manual(true).now_or_never().unwrap();
470        let token = result.unwrap().unwrap();
471        assert_eq!(token.to_string(), "x");
472    }
473
474    #[test]
475    fn parser_take_token_manual_not_command_name() {
476        let mut lexer = Lexer::with_code("X");
477        #[allow(clippy::mutable_key_type)]
478        let mut aliases = AliasSet::new();
479        aliases.insert(HashEntry::new(
480            "X".to_string(),
481            "x".to_string(),
482            false,
483            Location::dummy("?"),
484        ));
485        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
486
487        let result = parser.take_token_manual(false).now_or_never().unwrap();
488        let token = result.unwrap().unwrap();
489        assert_eq!(token.to_string(), "X");
490    }
491
492    #[test]
493    fn parser_take_token_manual_not_literal() {
494        let mut lexer = Lexer::with_code(r"\X");
495        #[allow(clippy::mutable_key_type)]
496        let mut aliases = AliasSet::new();
497        aliases.insert(HashEntry::new(
498            "X".to_string(),
499            "x".to_string(),
500            false,
501            Location::dummy("?"),
502        ));
503        aliases.insert(HashEntry::new(
504            r"\X".to_string(),
505            "quoted".to_string(),
506            false,
507            Location::dummy("?"),
508        ));
509        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
510
511        let result = parser.take_token_manual(true).now_or_never().unwrap();
512        let token = result.unwrap().unwrap();
513        assert_eq!(token.to_string(), r"\X");
514    }
515
516    #[test]
517    fn parser_take_token_manual_operator() {
518        let mut lexer = Lexer::with_code(";");
519        #[allow(clippy::mutable_key_type)]
520        let mut aliases = AliasSet::new();
521        aliases.insert(HashEntry::new(
522            ";".to_string(),
523            "x".to_string(),
524            false,
525            Location::dummy("?"),
526        ));
527        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
528
529        let result = parser.take_token_manual(true).now_or_never().unwrap();
530        let token = result.unwrap().unwrap();
531        assert_eq!(token.id, Operator(super::super::lex::Operator::Semicolon));
532        assert_eq!(token.word.to_string_if_literal().unwrap(), ";");
533    }
534
535    #[test]
536    fn parser_take_token_manual_no_match() {
537        let mut lexer = Lexer::with_code("X");
538        let mut parser = Parser::new(&mut lexer);
539
540        let result = parser.take_token_manual(true).now_or_never().unwrap();
541        let token = result.unwrap().unwrap();
542        assert_eq!(token.to_string(), "X");
543    }
544
545    #[test]
546    fn parser_take_token_manual_recursive_substitution() {
547        let mut lexer = Lexer::with_code("X");
548        #[allow(clippy::mutable_key_type)]
549        let mut aliases = AliasSet::new();
550        aliases.insert(HashEntry::new(
551            "X".to_string(),
552            "Y x".to_string(),
553            false,
554            Location::dummy("?"),
555        ));
556        aliases.insert(HashEntry::new(
557            "Y".to_string(),
558            "X y".to_string(),
559            false,
560            Location::dummy("?"),
561        ));
562        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
563
564        let result = parser.take_token_manual(true).now_or_never().unwrap();
565        assert_matches!(result, Ok(Rec::AliasSubstituted));
566
567        let result = parser.take_token_manual(true).now_or_never().unwrap();
568        assert_matches!(result, Ok(Rec::AliasSubstituted));
569
570        let result = parser.take_token_manual(true).now_or_never().unwrap();
571        let token = result.unwrap().unwrap();
572        assert_eq!(token.to_string(), "X");
573
574        let result = parser.take_token_manual(true).now_or_never().unwrap();
575        let token = result.unwrap().unwrap();
576        assert_eq!(token.to_string(), "y");
577
578        let rec = parser.take_token_manual(true).now_or_never().unwrap();
579        let token = rec.unwrap().unwrap();
580        assert_eq!(token.to_string(), "x");
581    }
582
583    #[test]
584    fn parser_take_token_manual_after_blank_ending_substitution() {
585        let mut lexer = Lexer::with_code("X\tY");
586        #[allow(clippy::mutable_key_type)]
587        let mut aliases = AliasSet::new();
588        aliases.insert(HashEntry::new(
589            "X".to_string(),
590            " X ".to_string(),
591            false,
592            Location::dummy("?"),
593        ));
594        aliases.insert(HashEntry::new(
595            "Y".to_string(),
596            "y".to_string(),
597            false,
598            Location::dummy("?"),
599        ));
600        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
601
602        let result = parser.take_token_manual(true).now_or_never().unwrap();
603        assert_matches!(result, Ok(Rec::AliasSubstituted));
604
605        let result = parser.take_token_manual(true).now_or_never().unwrap();
606        let token = result.unwrap().unwrap();
607        assert_eq!(token.to_string(), "X");
608
609        let result = parser.take_token_manual(false).now_or_never().unwrap();
610        assert_matches!(result, Ok(Rec::AliasSubstituted));
611
612        let result = parser.take_token_manual(false).now_or_never().unwrap();
613        let token = result.unwrap().unwrap();
614        assert_eq!(token.to_string(), "y");
615    }
616
617    #[test]
618    fn parser_take_token_manual_not_after_blank_ending_substitution() {
619        let mut lexer = Lexer::with_code("X\tY");
620        #[allow(clippy::mutable_key_type)]
621        let mut aliases = AliasSet::new();
622        aliases.insert(HashEntry::new(
623            "X".to_string(),
624            " X".to_string(),
625            false,
626            Location::dummy("?"),
627        ));
628        aliases.insert(HashEntry::new(
629            "Y".to_string(),
630            "y".to_string(),
631            false,
632            Location::dummy("?"),
633        ));
634        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
635
636        let result = parser.take_token_manual(true).now_or_never().unwrap();
637        assert_matches!(result, Ok(Rec::AliasSubstituted));
638
639        let result = parser.take_token_manual(true).now_or_never().unwrap();
640        let token = result.unwrap().unwrap();
641        assert_eq!(token.to_string(), "X");
642
643        let result = parser.take_token_manual(false).now_or_never().unwrap();
644        let token = result.unwrap().unwrap();
645        assert_eq!(token.to_string(), "Y");
646    }
647
648    #[test]
649    fn parser_take_token_manual_global() {
650        let mut lexer = Lexer::with_code("X");
651        #[allow(clippy::mutable_key_type)]
652        let mut aliases = AliasSet::new();
653        aliases.insert(HashEntry::new(
654            "X".to_string(),
655            "x".to_string(),
656            true,
657            Location::dummy("?"),
658        ));
659        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
660
661        let result = parser.take_token_manual(false).now_or_never().unwrap();
662        assert_matches!(result, Ok(Rec::AliasSubstituted));
663
664        let result = parser.take_token_manual(false).now_or_never().unwrap();
665        let token = result.unwrap().unwrap();
666        assert_eq!(token.to_string(), "x");
667    }
668
669    #[test]
670    fn parser_take_token_auto_non_keyword() {
671        let mut lexer = Lexer::with_code("X");
672        #[allow(clippy::mutable_key_type)]
673        let mut aliases = AliasSet::new();
674        aliases.insert(HashEntry::new(
675            "X".to_string(),
676            "x".to_string(),
677            true,
678            Location::dummy("?"),
679        ));
680        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
681
682        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
683        assert_eq!(token.to_string(), "x");
684    }
685
686    #[test]
687    fn parser_take_token_auto_keyword_matched() {
688        let mut lexer = Lexer::with_code("if");
689        #[allow(clippy::mutable_key_type)]
690        let mut aliases = AliasSet::new();
691        aliases.insert(HashEntry::new(
692            "if".to_string(),
693            "x".to_string(),
694            true,
695            Location::dummy("?"),
696        ));
697        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
698
699        let token = parser
700            .take_token_auto(&[Keyword::If])
701            .now_or_never()
702            .unwrap()
703            .unwrap();
704        assert_eq!(token.to_string(), "if");
705    }
706
707    #[test]
708    fn parser_take_token_auto_keyword_unmatched() {
709        let mut lexer = Lexer::with_code("if");
710        #[allow(clippy::mutable_key_type)]
711        let mut aliases = AliasSet::new();
712        aliases.insert(HashEntry::new(
713            "if".to_string(),
714            "x".to_string(),
715            true,
716            Location::dummy("?"),
717        ));
718        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
719
720        let token = parser.take_token_auto(&[]).now_or_never().unwrap().unwrap();
721        assert_eq!(token.to_string(), "x");
722    }
723
724    #[test]
725    fn parser_take_token_auto_alias_substitution_to_keyword_matched() {
726        let mut lexer = Lexer::with_code("X");
727        #[allow(clippy::mutable_key_type)]
728        let mut aliases = AliasSet::new();
729        aliases.insert(HashEntry::new(
730            "X".to_string(),
731            "if".to_string(),
732            true,
733            Location::dummy("?"),
734        ));
735        aliases.insert(HashEntry::new(
736            "if".to_string(),
737            "x".to_string(),
738            true,
739            Location::dummy("?"),
740        ));
741        let mut parser = Parser::config().aliases(&aliases).input(&mut lexer);
742
743        let token = parser
744            .take_token_auto(&[Keyword::If])
745            .now_or_never()
746            .unwrap()
747            .unwrap();
748        assert_eq!(token.to_string(), "if");
749    }
750
751    #[test]
752    fn parser_has_blank_true() {
753        let mut lexer = Lexer::with_code(" ");
754        let mut parser = Parser::new(&mut lexer);
755        let result = parser.has_blank().now_or_never().unwrap();
756        assert_eq!(result, Ok(true));
757    }
758
759    #[test]
760    fn parser_has_blank_false() {
761        let mut lexer = Lexer::with_code("(");
762        let mut parser = Parser::new(&mut lexer);
763        let result = parser.has_blank().now_or_never().unwrap();
764        assert_eq!(result, Ok(false));
765    }
766
767    #[test]
768    fn parser_has_blank_eof() {
769        let mut lexer = Lexer::with_code("");
770        let mut parser = Parser::new(&mut lexer);
771        let result = parser.has_blank().now_or_never().unwrap();
772        assert_eq!(result, Ok(false));
773    }
774
775    #[test]
776    fn parser_has_blank_true_with_line_continuations() {
777        let mut lexer = Lexer::with_code("\\\n\\\n ");
778        let mut parser = Parser::new(&mut lexer);
779        let result = parser.has_blank().now_or_never().unwrap();
780        assert_eq!(result, Ok(true));
781    }
782
783    #[test]
784    fn parser_has_blank_false_with_line_continuations() {
785        let mut lexer = Lexer::with_code("\\\n\\\n\\\n(");
786        let mut parser = Parser::new(&mut lexer);
787        let result = parser.has_blank().now_or_never().unwrap();
788        assert_eq!(result, Ok(false));
789    }
790
791    #[test]
792    #[should_panic(expected = "There should be no pending token")]
793    fn parser_has_blank_with_pending_token() {
794        let mut lexer = Lexer::with_code("foo");
795        let mut parser = Parser::new(&mut lexer);
796        parser.peek_token().now_or_never().unwrap().unwrap();
797        let _ = parser.has_blank().now_or_never().unwrap();
798    }
799
800    #[test]
801    fn parser_reading_no_here_doc_contents() {
802        let mut lexer = Lexer::with_code("X");
803        let mut parser = Parser::new(&mut lexer);
804        parser.here_doc_contents().now_or_never().unwrap().unwrap();
805
806        let location = lexer.location().now_or_never().unwrap().unwrap();
807        assert_eq!(location.code.start_line_number.get(), 1);
808        assert_eq!(location.range, 0..1);
809    }
810
811    #[test]
812    fn parser_reading_one_here_doc_content() {
813        let delimiter = "END".parse().unwrap();
814
815        let mut lexer = Lexer::with_code("END\nX");
816        let mut parser = Parser::new(&mut lexer);
817        let remove_tabs = false;
818        let here_doc = Rc::new(HereDoc {
819            delimiter,
820            remove_tabs,
821            content: OnceCell::new(),
822        });
823        parser.memorize_unread_here_doc(Rc::clone(&here_doc));
824        parser.here_doc_contents().now_or_never().unwrap().unwrap();
825        assert_eq!(here_doc.delimiter.to_string(), "END");
826        assert_eq!(here_doc.remove_tabs, remove_tabs);
827        assert_eq!(here_doc.content.get().unwrap().0, []);
828
829        let location = lexer.location().now_or_never().unwrap().unwrap();
830        assert_eq!(location.code.start_line_number.get(), 1);
831        assert_eq!(location.range, 4..5);
832    }
833
834    #[test]
835    fn parser_reading_many_here_doc_contents() {
836        let delimiter1 = "ONE".parse().unwrap();
837        let delimiter2 = "TWO".parse().unwrap();
838        let delimiter3 = "THREE".parse().unwrap();
839
840        let mut lexer = Lexer::with_code("1\nONE\nTWO\n3\nTHREE\nX");
841        let mut parser = Parser::new(&mut lexer);
842        let here_doc1 = Rc::new(HereDoc {
843            delimiter: delimiter1,
844            remove_tabs: false,
845            content: OnceCell::new(),
846        });
847        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
848        let here_doc2 = Rc::new(HereDoc {
849            delimiter: delimiter2,
850            remove_tabs: true,
851            content: OnceCell::new(),
852        });
853        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
854        let here_doc3 = Rc::new(HereDoc {
855            delimiter: delimiter3,
856            remove_tabs: false,
857            content: OnceCell::new(),
858        });
859        parser.memorize_unread_here_doc(Rc::clone(&here_doc3));
860        parser.here_doc_contents().now_or_never().unwrap().unwrap();
861        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
862        assert_eq!(here_doc1.remove_tabs, false);
863        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
864        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
865        assert_eq!(here_doc2.remove_tabs, true);
866        assert_eq!(here_doc2.content.get().unwrap().to_string(), "");
867        assert_eq!(here_doc3.delimiter.to_string(), "THREE");
868        assert_eq!(here_doc3.remove_tabs, false);
869        assert_eq!(here_doc3.content.get().unwrap().to_string(), "3\n");
870    }
871
872    #[test]
873    fn parser_reading_here_doc_contents_twice() {
874        let delimiter1 = "ONE".parse().unwrap();
875        let delimiter2 = "TWO".parse().unwrap();
876
877        let mut lexer = Lexer::with_code("1\nONE\n2\nTWO\n");
878        let mut parser = Parser::new(&mut lexer);
879        let here_doc1 = Rc::new(HereDoc {
880            delimiter: delimiter1,
881            remove_tabs: false,
882            content: OnceCell::new(),
883        });
884        parser.memorize_unread_here_doc(Rc::clone(&here_doc1));
885        parser.here_doc_contents().now_or_never().unwrap().unwrap();
886        let here_doc2 = Rc::new(HereDoc {
887            delimiter: delimiter2,
888            remove_tabs: true,
889            content: OnceCell::new(),
890        });
891        parser.memorize_unread_here_doc(Rc::clone(&here_doc2));
892        parser.here_doc_contents().now_or_never().unwrap().unwrap();
893        assert_eq!(here_doc1.delimiter.to_string(), "ONE");
894        assert_eq!(here_doc1.remove_tabs, false);
895        assert_eq!(here_doc1.content.get().unwrap().to_string(), "1\n");
896        assert_eq!(here_doc2.delimiter.to_string(), "TWO");
897        assert_eq!(here_doc2.remove_tabs, true);
898        assert_eq!(here_doc2.content.get().unwrap().to_string(), "2\n");
899    }
900
901    #[test]
902    #[should_panic(expected = "No token must be peeked before reading here-doc contents")]
903    fn parser_here_doc_contents_must_be_called_without_pending_token() {
904        let mut lexer = Lexer::with_code("X");
905        let mut parser = Parser::new(&mut lexer);
906        parser.peek_token().now_or_never().unwrap().unwrap();
907        parser.here_doc_contents().now_or_never().unwrap().unwrap();
908    }
909}
yash_syntax/parser/core.rs

yash_syntax/parser/
core.rs
