yash_syntax/parser/lex/
core.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Fundamental building blocks for the lexical analyzer
18
19use super::keyword::Keyword;
20use super::op::Operator;
21use crate::alias::Alias;
22use crate::input::Context;
23use crate::input::InputObject;
24use crate::input::Memory;
25use crate::parser::core::Result;
26use crate::parser::error::Error;
27use crate::source::Code;
28use crate::source::Location;
29use crate::source::Source;
30use crate::source::SourceChar;
31use crate::source::source_chars;
32use crate::syntax::Word;
33use std::cell::RefCell;
34use std::fmt;
35use std::num::NonZeroU64;
36use std::ops::Deref;
37use std::ops::DerefMut;
38use std::ops::Range;
39use std::pin::Pin;
40use std::rc::Rc;
41
42/// Returns true if the character is a blank character.
43pub fn is_blank(c: char) -> bool {
44    // TODO locale
45    c != '\n' && c.is_whitespace()
46}
47
48/// Result of [`LexerCore::peek_char`]
49#[derive(Clone, Copy, Debug, Eq, PartialEq)]
50enum PeekChar<'a> {
51    Char(&'a SourceChar),
52    EndOfInput(&'a Location),
53}
54
55impl<'a> PeekChar<'a> {
56    /// Returns the location that was peeked.
57    #[must_use]
58    fn location<'b>(self: &'b PeekChar<'a>) -> &'a Location {
59        match self {
60            PeekChar::Char(c) => &c.location,
61            PeekChar::EndOfInput(l) => l,
62        }
63    }
64}
65
66/// Token identifier, or classification of tokens
67///
68/// This enum classifies a token as defined in POSIX XCU 2.10.1 Shell Grammar Lexical
69/// Conventions, but does not exactly reflect further distinction defined in
70/// POSIX XCU 2.10.2 Shell Grammar Rules.
71///
72/// For convenience, the special token identifier `EndOfInput` is included.
73#[derive(Clone, Copy, Debug, Eq, PartialEq)]
74pub enum TokenId {
75    /// `TOKEN`
76    ///
77    /// If this token _looks like_ a reserved word, this variant has some
78    /// associated `Keyword` that describes the word. However, it depends on
79    /// context whether a token is actually regarded as a reserved word or
80    /// just as an ordinary word. You must ensure that you're in an
81    /// applicable context when examining the `Keyword` value.
82    Token(Option<Keyword>),
83    /// Operator
84    Operator(Operator),
85    /// `IO_NUMBER`
86    IoNumber,
87    /// `IO_LOCATION`
88    IoLocation,
89    /// Imaginary token identifier for the end of input
90    EndOfInput,
91}
92
93impl TokenId {
94    /// Determines if this token can be a delimiter of a clause.
95    ///
96    /// This function delegates to [`Keyword::is_clause_delimiter`] if the token
97    /// ID is a (possible) keyword, or to [`Operator::is_clause_delimiter`] if
98    /// it is an operator. For `EndOfInput` the function returns true.
99    /// Otherwise, the result is false.
100    pub fn is_clause_delimiter(self) -> bool {
101        use TokenId::*;
102        match self {
103            Token(Some(keyword)) => keyword.is_clause_delimiter(),
104            Token(None) => false,
105            Operator(operator) => operator.is_clause_delimiter(),
106            IoNumber => false,
107            IoLocation => false,
108            EndOfInput => true,
109        }
110    }
111}
112
113/// Result of lexical analysis produced by the [`Lexer`]
114#[derive(Debug)]
115pub struct Token {
116    /// Content of the token
117    ///
118    /// The word value contains at least one [unit](crate::syntax::WordUnit),
119    /// regardless of whether the token is an operator. The only exception is
120    /// when `id` is `EndOfInput`, in which case the word is empty.
121    pub word: Word,
122    /// Token identifier
123    pub id: TokenId,
124    /// Position of the first character of the word
125    pub index: usize,
126}
127
128impl fmt::Display for Token {
129    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130        write!(f, "{}", self.word)
131    }
132}
133
134/// State of the input function in a lexer
135#[derive(Clone, Debug)]
136enum InputState {
137    Alive,
138    EndOfInput(Location),
139    Error(Error),
140}
141
142/// Source character with additional attribute
143#[derive(Clone, Debug, Eq, PartialEq)]
144struct SourceCharEx {
145    value: SourceChar,
146    is_line_continuation: bool,
147}
148
149fn ex<I: IntoIterator<Item = SourceChar>>(i: I) -> impl Iterator<Item = SourceCharEx> {
150    i.into_iter().map(|sc| SourceCharEx {
151        value: sc,
152        is_line_continuation: false,
153    })
154}
155
156/// Core part of the lexical analyzer
157struct LexerCore<'a> {
158    // The `input` field could be a `&'a mut dyn InputObject + 'a`, but it is
159    // `Box<dyn InputObject + 'a>` to allow the lexer to take ownership of the
160    // input object. This is necessary for `Lexer::with_code` and similarly
161    // constructed lexers.
162    input: Box<dyn InputObject + 'a>,
163    state: InputState,
164    raw_code: Rc<Code>,
165    source: Vec<SourceCharEx>,
166    index: usize,
167    mode: yash_env::parser::Mode,
168}
169
170impl<'a> LexerCore<'a> {
171    /// Creates a new lexer core that reads using the given input function.
172    #[must_use]
173    fn new(
174        input: Box<dyn InputObject + 'a>,
175        start_line_number: NonZeroU64,
176        source: Rc<Source>,
177    ) -> LexerCore<'a> {
178        LexerCore {
179            input,
180            raw_code: Rc::new(Code {
181                value: RefCell::new(String::new()),
182                start_line_number,
183                source,
184            }),
185            state: InputState::Alive,
186            source: Vec::new(),
187            index: 0,
188            mode: yash_env::parser::Mode::default(),
189        }
190    }
191
192    /// Computes the start index of the location at the current position.
193    #[must_use]
194    fn next_index(&self) -> usize {
195        let Some(last) = self.source.last() else {
196            return 0;
197        };
198
199        let mut location = &last.value.location;
200        while let Source::Alias { original, .. } = &*location.code.source {
201            location = original;
202        }
203        location.range.end
204    }
205
206    /// Peeks the next character, reading the next line if necessary.
207    async fn peek_char(&mut self) -> Result<PeekChar<'_>> {
208        loop {
209            // if let Some(sc) = self.source.get(self.index) {
210            //     return Ok(PeekChar::Char(&sc.value));
211            if self.index < self.source.len() {
212                return Ok(PeekChar::Char(&self.source[self.index].value));
213            }
214
215            match self.state {
216                InputState::Alive => (),
217                InputState::EndOfInput(ref location) => return Ok(PeekChar::EndOfInput(location)),
218                InputState::Error(ref error) => return Err(error.clone()),
219            }
220
221            // Read more input
222            let index = self.next_index();
223            match self.input.next_line(&self.input_context()).await {
224                Ok(line) => {
225                    if line.is_empty() {
226                        // End of input
227                        self.state = InputState::EndOfInput(Location {
228                            code: Rc::clone(&self.raw_code),
229                            range: index..index,
230                        });
231                    } else {
232                        // Successful read
233                        self.raw_code.value.borrow_mut().push_str(&line);
234                        self.source
235                            .extend(ex(source_chars(&line, &self.raw_code, index)));
236                    }
237                }
238                Err(io_error) => {
239                    self.state = InputState::Error(Error {
240                        cause: io_error.into(),
241                        location: Location {
242                            code: Rc::clone(&self.raw_code),
243                            range: index..index,
244                        },
245                    });
246                }
247            }
248        }
249    }
250
251    /// Returns the input context for the next character.
252    fn input_context(&self) -> Context {
253        let mut context = Context::default();
254        context.set_is_first_line(self.raw_code.value.borrow().is_empty());
255        context
256    }
257
258    /// Consumes the next character.
259    ///
260    /// This function must be called after [`peek_char`](Lexer::peek_char) has successfully
261    /// returned the character. Consuming a character that has not yet been peeked would result
262    /// in a panic!
263    fn consume_char(&mut self) {
264        assert!(
265            self.index < self.source.len(),
266            "A character must have been peeked before being consumed: index={}",
267            self.index
268        );
269        self.index += 1;
270    }
271
272    /// Returns a reference to the character at the given index.
273    #[must_use]
274    fn peek_char_at(&self, index: usize) -> &SourceChar {
275        assert!(
276            index <= self.index,
277            "The index {} must not be larger than the current index {}",
278            index,
279            self.index
280        );
281        &self.source[index].value
282    }
283
284    /// Returns the current index.
285    #[must_use]
286    fn index(&self) -> usize {
287        self.index
288    }
289
290    /// Rewinds the index to the given value.
291    fn rewind(&mut self, index: usize) {
292        assert!(
293            index <= self.index,
294            "The new index {} must not be larger than the current index {}",
295            index,
296            self.index
297        );
298        self.index = index;
299    }
300
301    /// Checks if there is any character that has been read from the input
302    /// source but not yet consumed.
303    #[must_use]
304    fn pending(&self) -> bool {
305        self.index < self.source.len()
306    }
307
308    /// Clears the internal buffer.
309    fn flush(&mut self) {
310        let start_line_number = self.raw_code.line_number(usize::MAX);
311        self.raw_code = Rc::new(Code {
312            value: RefCell::new(String::new()),
313            start_line_number,
314            source: self.raw_code.source.clone(),
315        });
316        self.source.clear();
317        self.index = 0;
318    }
319
320    /// Clears an end-of-input or error status so that the lexer can resume
321    /// parsing.
322    fn reset(&mut self) {
323        self.state = InputState::Alive;
324        self.flush();
325    }
326
327    /// Extracts a string from the source code range.
328    fn source_string(&self, range: Range<usize>) -> String {
329        self.source[range].iter().map(|c| c.value.value).collect()
330    }
331
332    /// Returns a location for a given range of the source code.
333    #[must_use]
334    fn location_range(&self, range: Range<usize>) -> Location {
335        if range.start == self.source.len()
336            && let InputState::EndOfInput(ref location) = self.state
337        {
338            return location.clone();
339        }
340        let start = &self.peek_char_at(range.start).location;
341        let code = start.code.clone();
342        let end = range
343            .map(|index| &self.peek_char_at(index).location)
344            .take_while(|location| location.code == code)
345            .last()
346            .map(|location| location.range.end)
347            .unwrap_or(start.range.start);
348        let range = start.range.start..end;
349        Location { code, range }
350    }
351
352    /// Marks the characters in the given range as line continuation.
353    ///
354    /// This function sets the `is_line_continuation` flag of the characters in
355    /// the range to true. The characters must have been read before calling
356    /// this function.
357    fn mark_line_continuation(&mut self, range: Range<usize>) {
358        assert!(
359            range.end <= self.index,
360            "characters must have been read (range = {:?}, current index = {})",
361            range,
362            self.index
363        );
364        for sc in &mut self.source[range] {
365            sc.is_line_continuation = true;
366        }
367    }
368
369    /// Performs alias substitution.
370    ///
371    /// This function replaces the characters starting from the `begin` index up
372    /// to the current position with the alias value. The resulting part of code
373    /// will be characters with a [`Source::Alias`] origin.
374    fn substitute_alias(&mut self, begin: usize, alias: &Rc<Alias>) {
375        let end = self.index;
376        assert!(
377            begin < end,
378            "begin index {begin} should be less than end index {end}"
379        );
380
381        let source = Rc::new(Source::Alias {
382            original: self.location_range(begin..end),
383            alias: alias.clone(),
384        });
385        let code = Rc::new(Code {
386            value: RefCell::new(alias.replacement.clone()),
387            start_line_number: NonZeroU64::new(1).unwrap(),
388            source,
389        });
390        let repl = ex(source_chars(&alias.replacement, &code, 0));
391
392        self.source.splice(begin..end, repl);
393        self.index = begin;
394    }
395
396    /// Tests if the given index is after the replacement string of alias
397    /// substitution that ends with a blank.
398    ///
399    /// # Panics
400    ///
401    /// If `index` is larger than the currently read index.
402    fn is_after_blank_ending_alias(&self, index: usize) -> bool {
403        fn ends_with_blank(s: &str) -> bool {
404            s.chars().next_back().is_some_and(is_blank)
405        }
406        fn is_same_alias(alias: &Alias, sc: Option<&SourceCharEx>) -> bool {
407            sc.is_some_and(|sc| sc.value.location.code.source.is_alias_for(&alias.name))
408        }
409
410        for index in (0..index).rev() {
411            let sc = &self.source[index];
412
413            if !sc.is_line_continuation && !is_blank(sc.value.value) {
414                return false;
415            }
416
417            if let Source::Alias { ref alias, .. } = *sc.value.location.code.source
418                && ends_with_blank(&alias.replacement)
419                && !is_same_alias(alias, self.source.get(index + 1))
420            {
421                return true;
422            }
423        }
424
425        false
426    }
427}
428
429impl fmt::Debug for LexerCore<'_> {
430    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
431        f.debug_struct("LexerCore")
432            .field("state", &self.state)
433            .field("source", &self.source)
434            .field("index", &self.index)
435            .finish_non_exhaustive()
436    }
437}
438
439/// Configuration for the [lexer](Lexer)
440///
441/// `Config` is a builder for the lexer. A [new](Self::new) instance is created
442/// with default settings. You can then customize the settings by modifying the
443/// corresponding fields. Finally, you can pass an input object to the
444/// [`input`](Self::input) method to create a lexer.
445///
446/// # Deprecation
447///
448/// This struct is deprecated. Use [`yash_env::parser::Config`] instead.
449#[deprecated(since = "0.17.0", note = "use `yash_env::parser::Config` instead")]
450#[derive(Debug)]
451#[must_use = "you must call `input` to create a lexer"]
452#[non_exhaustive]
453pub struct Config {
454    /// Line number for the first line of the input
455    ///
456    /// The lexer counts the line number from this value to annotate the
457    /// location of the tokens. The line number is saved in the
458    /// `start_line_number` field of the [`Code`] instance that is contained in
459    /// the [`Location`] instance of the token.
460    ///
461    /// The default value is 1.
462    pub start_line_number: NonZeroU64,
463
464    /// Source of the input
465    ///
466    /// The source is used to annotate the location of the tokens. This value
467    /// is saved in the `source` field of the [`Code`] instance that is
468    /// contained in the [`Location`] instance of the token.
469    ///
470    /// The default value is `None`, in which case the source is set to
471    /// [`Source::Unknown`]. It is recommended to set this to a more informative
472    /// value, so that the locations in the parsed syntax tree can be traced
473    /// back to the source code. Especially, the correct source is necessary to
474    /// indicate the location of possible errors that occur during parsing and
475    /// execution.
476    pub source: Option<Rc<Source>>,
477}
478
479#[allow(deprecated, reason = "for backward compatible API")]
480impl Config {
481    /// Creates a new configuration with default settings.
482    ///
483    /// # Deprecation
484    ///
485    /// This struct is deprecated. Use [`yash_env::parser::Config`] instead.
486    #[deprecated(since = "0.17.0", note = "use `yash_env::parser::Config` instead")]
487    pub fn new() -> Self {
488        Config {
489            start_line_number: NonZeroU64::MIN,
490            source: None,
491        }
492    }
493
494    /// Creates a lexer with the given input object.
495    pub fn input<'a>(self, input: Box<dyn InputObject + 'a>) -> Lexer<'a> {
496        let mut config = yash_env::parser::Config::with_input(input);
497        config.start_line_number = self.start_line_number;
498        config.source = self.source;
499        config.into()
500    }
501}
502
503#[allow(deprecated, reason = "for backward compatible API")]
504impl Default for Config {
505    fn default() -> Self {
506        Self::new()
507    }
508}
509
510/// Lexical analyzer
511///
512/// A lexer reads lines using an input function and parses the characters into tokens. It has an
513/// internal buffer containing the characters that have been read and the position (or the
514/// index) of the character that is to be parsed next.
515///
516/// `Lexer` has primitive functions such as [`peek_char`](Lexer::peek_char) that provide access
517/// to the character at the current position. Derived functions such as
518/// [`skip_blanks_and_comment`](Lexer::skip_blanks_and_comment) depend on those primitives to
519/// parse more complex structures in the source code. Usually, the lexer is used by a
520/// [parser](super::super::Parser) to read the source code and produce a syntax
521/// tree, so you don't need to call these functions directly.
522///
523/// To construct a lexer, create a configuration object
524/// ([`yash_env::parser::Config`]), set the desired fields, and then call
525/// `into()` or [`Lexer::from`].
526/// `Lexer` has several convenience functions such as [`new`](Self::new) and
527/// [`with_code`](Self::with_code) for creating a lexer with minimal
528/// configuration.
529///
530/// ```
531/// # use yash_env::parser::Config;
532/// # use yash_syntax::input::Memory;
533/// # use yash_syntax::parser::{lex::Lexer, Parser};
534/// # use yash_syntax::source::Source;
535/// let mut config = Config::with_input(Box::new(Memory::new("echo hello\n")));
536/// config.start_line_number = 10.try_into().unwrap();
537/// config.source = Some(Source::CommandString.into());
538/// let mut lexer = Lexer::from(config);
539/// let mut parser = Parser::new(&mut lexer);
540/// _ = parser.command_line();
541/// ```
542#[derive(Debug)]
543#[must_use]
544pub struct Lexer<'a> {
545    // `Lexer` is a thin wrapper around `LexerCore`. `Lexer` delegates most
546    // functions to `LexerCore`. `Lexer` adds automatic line-continuation
547    // skipping to `LexerCore`.
548    core: LexerCore<'a>,
549    line_continuation_enabled: bool,
550}
551
552/// Creates a lexer from a configuration defined in the [`yash-env`](yash_env)
553/// crate.
554impl<'a> From<yash_env::parser::Config<'a>> for Lexer<'a> {
555    fn from(config: yash_env::parser::Config<'a>) -> Self {
556        let input = config.input;
557        let start_line_number = config.start_line_number;
558        let source = config.source.unwrap_or_else(|| Rc::new(Source::Unknown));
559        let mut core = LexerCore::new(input, start_line_number, source);
560        core.mode = config.mode;
561        Lexer {
562            core,
563            line_continuation_enabled: true,
564        }
565    }
566}
567
568impl<'a> Lexer<'a> {
569    /// Creates a new configuration with default settings.
570    ///
571    /// This is a synonym for [`Config::new`]. You can modify the settings and
572    /// then create a lexer with the [`input`](Config::input) method.
573    ///
574    /// # Deprecation
575    ///
576    /// The `Config` struct defined in this module is deprecated. Use
577    /// [`yash_env::parser::Config`] instead.
578    #[allow(deprecated, reason = "for backward compatible API")]
579    #[deprecated(since = "0.17.0", note = "use `yash_env::parser::Config` instead")]
580    #[inline(always)]
581    pub fn config() -> Config {
582        Config::new()
583    }
584
585    /// Creates a new lexer that reads using the given input function.
586    ///
587    /// This is a convenience function that creates a lexer with the given input
588    /// object and the default configuration. To customize the configuration,
589    /// instantiate a [`yash_env::parser::Config`] object instead.
590    ///
591    /// This function is best used for testing or for simple cases where you
592    /// don't need to customize the lexer. For practical use, it is recommended
593    /// to provide supplementary information with a configuration before
594    /// creating a lexer.
595    pub fn new(input: Box<dyn InputObject + 'a>) -> Lexer<'a> {
596        yash_env::parser::Config::with_input(input).into()
597    }
598
599    /// Creates a new lexer with a fixed source code.
600    ///
601    /// This is a convenience function that creates a lexer that reads from a
602    /// string using [`Memory`] with the default configuration.
603    ///
604    /// This function is best used for testing or for simple cases where you
605    /// don't need to customize the lexer. For practical use, it is recommended
606    /// to provide supplementary information with a configuration before
607    /// creating a lexer.
608    pub fn with_code(code: &'a str) -> Lexer<'a> {
609        Self::new(Box::new(Memory::new(code)))
610    }
611
612    /// Creates a new lexer with a fixed source code.
613    ///
614    /// This is a convenience function that creates a lexer that reads from a
615    /// string using [`Memory`] with the specified source starting from line
616    /// number 1.
617    ///
618    /// This function is soft-deprecated. Use [`with_code`](Self::with_code)
619    /// instead if the source is `Unknown`. Otherwise, use
620    /// [`yash_env::parser::Config`] to create a lexer with a customized
621    /// configuration.
622    pub fn from_memory<S: Into<Rc<Source>>>(code: &'a str, source: S) -> Lexer<'a> {
623        fn inner(code: &str, source: Rc<Source>) -> Lexer<'_> {
624            let mut config = yash_env::parser::Config::with_input(Box::new(Memory::new(code)));
625            config.source = Some(source);
626            config.into()
627        }
628        inner(code, source.into())
629    }
630
631    /// Returns the parsing mode of this lexer.
632    ///
633    /// The mode reflects shell options that affect which syntax the parser
634    /// accepts (see [`yash_env::parser::Mode`]). Lexer and parser code consult
635    /// it to decide whether to accept or reject non-portable constructs. Lexers
636    /// created without an explicit mode permit all syntax.
637    #[must_use]
638    pub fn mode(&self) -> yash_env::parser::Mode {
639        self.core.mode
640    }
641
642    /// Sets the parsing mode of this lexer.
643    ///
644    /// A read-eval loop typically calls this before parsing each command line so
645    /// that changes to shell options (for example, via the `set` built-in) take
646    /// effect on subsequent parsing.
647    pub fn set_mode(&mut self, mode: yash_env::parser::Mode) {
648        self.core.mode = mode;
649    }
650
651    /// Disables line continuation recognition onward.
652    ///
653    /// By default, [`peek_char`](Self::peek_char) silently skips line
654    /// continuation sequences. When line continuation is disabled, however,
655    /// `peek_char` returns characters literally.
656    ///
657    /// Call [`enable_line_continuation`](Self::enable_line_continuation) to
658    /// switch line continuation recognition on.
659    ///
660    /// This function will panic if line continuation has already been disabled.
661    pub fn disable_line_continuation<'b>(&'b mut self) -> PlainLexer<'b, 'a> {
662        assert!(
663            self.line_continuation_enabled,
664            "line continuation already disabled"
665        );
666        self.line_continuation_enabled = false;
667        PlainLexer { lexer: self }
668    }
669
670    /// Re-enables line continuation.
671    ///
672    /// You can pass the `PlainLexer` returned from
673    /// [`disable_line_continuation`](Self::disable_line_continuation) to this
674    /// function to re-enable line continuation. That is equivalent to dropping
675    /// the `PlainLexer` instance, but the code will be more descriptive.
676    pub fn enable_line_continuation<'b>(_: PlainLexer<'a, 'b>) {}
677
678    /// Skips line continuation, i.e., a backslash followed by a newline.
679    ///
680    /// If there is a line continuation at the current position, this function
681    /// consumes the backslash and the newline and returns `Ok(true)`. The
682    /// characters are marked as line continuation.
683    ///
684    /// If there is no line continuation, this function does nothing and returns
685    /// `Ok(false)`.
686    ///
687    /// This function does nothing if line continuation has been
688    /// [disabled](Self::disable_line_continuation).
689    async fn line_continuation(&mut self) -> Result<bool> {
690        if !self.line_continuation_enabled {
691            return Ok(false);
692        }
693
694        let index = self.core.index();
695        match self.core.peek_char().await? {
696            PeekChar::Char(c) if c.value == '\\' => self.core.consume_char(),
697            _ => return Ok(false),
698        }
699
700        match self.core.peek_char().await? {
701            PeekChar::Char(c) if c.value == '\n' => self.core.consume_char(),
702            _ => {
703                self.core.rewind(index);
704                return Ok(false);
705            }
706        }
707
708        self.core.mark_line_continuation(index..index + 2);
709
710        Ok(true)
711    }
712
713    /// Peeks the next character.
714    ///
715    /// If the end of input is reached, `Ok(None)` is returned. On error,
716    /// `Err(_)` is returned.
717    ///
718    /// If line continuation recognition is enabled, combinations of a backslash
719    /// and a newline are silently skipped before returning the next character.
720    /// Call [`disable_line_continuation`](Self::disable_line_continuation) to
721    /// switch off line continuation recognition.
722    ///
723    /// This function requires a mutable reference to `self` since it may need
724    /// to read the next line if needed.
725    pub async fn peek_char(&mut self) -> Result<Option<char>> {
726        while self.line_continuation().await? {}
727
728        match self.core.peek_char().await? {
729            PeekChar::Char(source_char) => Ok(Some(source_char.value)),
730            PeekChar::EndOfInput(_) => Ok(None),
731        }
732    }
733
734    /// Returns the location of the next character.
735    ///
736    /// If there is no more character (that is, it is the end of input), an imaginary location
737    /// is returned that would be returned if a character existed.
738    ///
739    /// This function requires a mutable reference to `self` since it needs to
740    /// [peek](Self::peek_char) the next character.
741    pub async fn location(&mut self) -> Result<&Location> {
742        self.core.peek_char().await.map(|p| p.location())
743    }
744
745    /// Consumes the next character.
746    ///
747    /// This function must be called after [`peek_char`](Lexer::peek_char) has successfully
748    /// returned the character. Consuming a character that has not yet been peeked would result
749    /// in a panic!
750    pub fn consume_char(&mut self) {
751        self.core.consume_char()
752    }
753
754    /// Returns the position of the next character, counted from zero.
755    ///
756    /// ```
757    /// # use yash_syntax::parser::lex::Lexer;
758    /// # futures_executor::block_on(async {
759    /// let mut lexer = Lexer::with_code("abc");
760    /// assert_eq!(lexer.index(), 0);
761    /// let _ = lexer.peek_char().await;
762    /// assert_eq!(lexer.index(), 0);
763    /// lexer.consume_char();
764    /// assert_eq!(lexer.index(), 1);
765    /// # })
766    /// ```
767    #[must_use]
768    pub fn index(&self) -> usize {
769        self.core.index()
770    }
771
772    /// Moves the current position back to the given index so that characters that have been
773    /// consumed can be read again.
774    ///
775    /// The given index must not be larger than the [current index](Lexer::index), or this
776    /// function would panic.
777    ///
778    /// ```
779    /// # use yash_syntax::parser::lex::Lexer;
780    /// # futures_executor::block_on(async {
781    /// let mut lexer = Lexer::with_code("abc");
782    /// let saved_index = lexer.index();
783    /// assert_eq!(lexer.peek_char().await, Ok(Some('a')));
784    /// lexer.consume_char();
785    /// assert_eq!(lexer.peek_char().await, Ok(Some('b')));
786    /// lexer.rewind(saved_index);
787    /// assert_eq!(lexer.peek_char().await, Ok(Some('a')));
788    /// # })
789    /// ```
790    pub fn rewind(&mut self, index: usize) {
791        self.core.rewind(index)
792    }
793
794    /// Checks if there is any character that has been read from the input
795    /// source but not yet consumed.
796    #[must_use]
797    pub fn pending(&self) -> bool {
798        self.core.pending()
799    }
800
801    /// Clears the internal buffer of the lexer.
802    ///
803    /// Locations returned from [`location`](Self::location) share a single code
804    /// instance that is also retained by the lexer. The code grows long as the
805    /// lexer reads more input. To prevent the code from getting too large, you
806    /// can call this function that replaces the retained code with a new empty
807    /// one. The new code's `start_line_number` will be incremented by the
808    /// number of lines in the previous.
809    pub fn flush(&mut self) {
810        self.core.flush()
811    }
812
813    /// Clears an end-of-input or error status so that the lexer can resume
814    /// parsing.
815    ///
816    /// This function will be useful only in an interactive shell where the user
817    /// can continue entering commands even after (s)he sends an end-of-input or
818    /// is interrupted by a syntax error.
819    pub fn reset(&mut self) {
820        self.core.reset()
821    }
822
823    /// Peeks the next character and, if the given decider function returns true for it,
824    /// advances the position.
825    ///
826    /// Returns the consumed character if the function returned true. Returns `Ok(None)` if it
827    /// returned false or there is no more character.
828    pub async fn consume_char_if<F>(&mut self, mut f: F) -> Result<Option<&SourceChar>>
829    where
830        F: FnMut(char) -> bool,
831    {
832        self.consume_char_if_dyn(&mut f).await
833    }
834
835    /// Dynamic version of [`Self::consume_char_if`].
836    pub(crate) async fn consume_char_if_dyn(
837        &mut self,
838        f: &mut dyn FnMut(char) -> bool,
839    ) -> Result<Option<&SourceChar>> {
840        match self.peek_char().await? {
841            Some(c) if f(c) => {
842                let index = self.index();
843                self.consume_char();
844                Ok(Some(self.core.peek_char_at(index)))
845            }
846            _ => Ok(None),
847        }
848    }
849
850    /// Extracts a string from the source code range.
851    ///
852    /// This function returns the source code string for the range specified by
853    /// the argument. The range must specify a valid index. If the index points
854    /// to a character that have not yet read, this function will panic!.
855    ///
856    /// # Panics
857    ///
858    /// If the argument index is out of bounds, i.e., pointing to an unread
859    /// character.
860    #[inline]
861    pub fn source_string(&self, range: Range<usize>) -> String {
862        self.core.source_string(range)
863    }
864
865    /// Returns a location for a given range of the source code.
866    ///
867    /// All the characters in the range must have been
868    /// [consume](Self::consume_char)d. If the range refers to an unconsumed
869    /// character, this function will panic!
870    ///
871    /// If the characters are from more than one [`Code`] fragment, the location
872    /// will only cover the initial portion of the range sharing the same
873    /// `Code`.
874    ///
875    /// # Panics
876    ///
877    /// This function will panic if the range refers to an unconsumed character.
878    ///
879    /// If the start index of the range is the end of input, it must have been
880    /// peeked and the range must be empty, or the function will panic.
881    #[must_use]
882    pub fn location_range(&self, range: Range<usize>) -> Location {
883        self.core.location_range(range)
884    }
885
886    /// Performs alias substitution right before the current position.
887    ///
888    /// This function must be called just after a [word](WordLexer::word) has been parsed that
889    /// matches the name of the argument alias. No check is done in this function that there is
890    /// a matching word before the current position. The characters starting from the `begin`
891    /// index up to the current position are silently replaced with the alias value.
892    ///
893    /// The resulting part of code will be characters with a [`Source::Alias`] origin.
894    ///
895    /// After the substitution, the position will be set before the replaced string.
896    ///
897    /// # Panics
898    ///
899    /// If the replaced part is empty, i.e., `begin >= self.index()`.
900    pub fn substitute_alias(&mut self, begin: usize, alias: &Rc<Alias>) {
901        self.core.substitute_alias(begin, alias)
902    }
903
904    /// Tests if the given index is after the replacement string of alias
905    /// substitution that ends with a blank.
906    ///
907    /// # Panics
908    ///
909    /// If `index` is larger than the currently read index.
910    pub fn is_after_blank_ending_alias(&self, index: usize) -> bool {
911        self.core.is_after_blank_ending_alias(index)
912    }
913
914    /// Parses an optional compound list that is the content of a command
915    /// substitution.
916    ///
917    /// This function consumes characters until a token that cannot be the
918    /// beginning of an and-or list is found and returns the string that was
919    /// consumed.
920    pub async fn inner_program(&mut self) -> Result<String> {
921        let begin = self.index();
922
923        let mut parser = super::super::Parser::new(self);
924        parser.maybe_compound_list().await?;
925
926        let end = parser.peek_token().await?.index;
927        self.rewind(end);
928
929        Ok(self.core.source_string(begin..end))
930    }
931
932    /// Like [`Lexer::inner_program`], but returns the future in a pinning box.
933    pub fn inner_program_boxed(&mut self) -> Pin<Box<dyn Future<Output = Result<String>> + '_>> {
934        Box::pin(self.inner_program())
935    }
936}
937
938/// Reference to [`Lexer`] with line continuation disabled
939///
940/// This struct implements the RAII pattern for temporarily disabling line
941/// continuation. When you disable the line continuation of a lexer, you get an
942/// instance of `PlainLexer`. You can access the original lexer via the
943/// `PlainLexer` until you drop it, when the line continuation is automatically
944/// re-enabled.
945#[derive(Debug)]
946#[must_use = "You must retain the PlainLexer to keep line continuation disabled"]
947pub struct PlainLexer<'a, 'b> {
948    lexer: &'a mut Lexer<'b>,
949}
950
951impl<'b> Deref for PlainLexer<'_, 'b> {
952    type Target = Lexer<'b>;
953    fn deref(&self) -> &Lexer<'b> {
954        self.lexer
955    }
956}
957
958impl<'b> DerefMut for PlainLexer<'_, 'b> {
959    fn deref_mut(&mut self) -> &mut Lexer<'b> {
960        self.lexer
961    }
962}
963
964impl Drop for PlainLexer<'_, '_> {
965    fn drop(&mut self) {
966        self.lexer.line_continuation_enabled = true;
967    }
968}
969
970/// Context in which a [word](crate::syntax::Word) is parsed
971///
972/// The parse of the word of a [switch](crate::syntax::Switch) depends on
973/// whether the parameter expansion containing the switch is part of a text or a
974/// word. A `WordContext` value is used to decide the behavior of the lexer.
975///
976/// Parser functions that depend on the context are implemented in
977/// [`WordLexer`].
978#[derive(Clone, Copy, Debug, Eq, PartialEq)]
979pub enum WordContext {
980    /// The text unit being parsed is part of a [text](crate::syntax::Text).
981    Text,
982    /// The text unit being parsed is part of a [word](crate::syntax::Word).
983    Word,
984}
985
986/// Lexer with additional information for parsing [texts](crate::syntax::Text)
987/// and [words](crate::syntax::Word)
988#[derive(Debug)]
989pub struct WordLexer<'a, 'b> {
990    pub lexer: &'a mut Lexer<'b>,
991    pub context: WordContext,
992}
993
994impl<'b> Deref for WordLexer<'_, 'b> {
995    type Target = Lexer<'b>;
996    fn deref(&self) -> &Lexer<'b> {
997        self.lexer
998    }
999}
1000
1001impl<'b> DerefMut for WordLexer<'_, 'b> {
1002    fn deref_mut(&mut self) -> &mut Lexer<'b> {
1003        self.lexer
1004    }
1005}
1006
1007#[cfg(test)]
1008mod tests {
1009    use super::*;
1010    use crate::input::Input;
1011    use crate::parser::error::ErrorCause;
1012    use crate::parser::error::SyntaxError;
1013    use assert_matches::assert_matches;
1014    use futures_util::FutureExt as _;
1015
1016    #[test]
1017    fn lexer_mode_defaults_to_permissive() {
1018        let lexer = Lexer::with_code("");
1019        assert_eq!(lexer.mode(), yash_env::parser::Mode::default());
1020    }
1021
1022    #[test]
1023    fn lexer_mode_round_trips() {
1024        let mut lexer = Lexer::with_code("");
1025        let mut mode = yash_env::parser::Mode::default();
1026        mode.portable = true;
1027        lexer.set_mode(mode);
1028        assert_eq!(lexer.mode(), mode);
1029    }
1030
1031    #[test]
1032    fn lexer_from_config_carries_mode() {
1033        let mut config = yash_env::parser::Config::with_input(Box::new(Memory::new("")));
1034        config.mode.portable = true;
1035        let lexer = Lexer::from(config);
1036        assert!(lexer.mode().portable);
1037    }
1038
1039    #[test]
1040    fn lexer_core_peek_char_empty_source() {
1041        let input = Memory::new("");
1042        let line = NonZeroU64::new(32).unwrap();
1043        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1044        let result = lexer.peek_char().now_or_never().unwrap();
1045        assert_matches!(result, Ok(PeekChar::EndOfInput(location)) => {
1046            assert_eq!(*location.code.value.borrow(), "");
1047            assert_eq!(location.code.start_line_number, line);
1048            assert_eq!(*location.code.source, Source::Unknown);
1049            assert_eq!(location.range, 0..0);
1050        });
1051    }
1052
1053    #[test]
1054    fn lexer_core_peek_char_io_error() {
1055        #[derive(Debug)]
1056        struct Failing;
1057        impl fmt::Display for Failing {
1058            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1059                write!(f, "Failing")
1060            }
1061        }
1062        impl std::error::Error for Failing {}
1063        impl Input for Failing {
1064            async fn next_line(&mut self, _: &Context) -> crate::input::Result {
1065                Err(std::io::Error::other(Failing))
1066            }
1067        }
1068        let line = NonZeroU64::new(42).unwrap();
1069        let mut lexer = LexerCore::new(Box::new(Failing), line, Rc::new(Source::Unknown));
1070
1071        let e = lexer.peek_char().now_or_never().unwrap().unwrap_err();
1072        assert_matches!(e.cause, ErrorCause::Io(io_error) => {
1073            assert_eq!(io_error.kind(), std::io::ErrorKind::Other);
1074        });
1075        assert_eq!(*e.location.code.value.borrow(), "");
1076        assert_eq!(e.location.code.start_line_number, line);
1077        assert_eq!(*e.location.code.source, Source::Unknown);
1078        assert_eq!(e.location.range, 0..0);
1079    }
1080
1081    #[test]
1082    fn lexer_core_peek_char_context_is_first_line() {
1083        // In this test case, this mock input function will be called twice.
1084        struct InputMock {
1085            first: bool,
1086        }
1087        impl Input for InputMock {
1088            async fn next_line(&mut self, context: &Context) -> crate::input::Result {
1089                assert_eq!(context.is_first_line(), self.first);
1090                self.first = false;
1091                Ok("\n".to_owned())
1092            }
1093        }
1094
1095        let input = InputMock { first: true };
1096        let line = NonZeroU64::new(42).unwrap();
1097        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1098
1099        let peek = lexer.peek_char().now_or_never().unwrap();
1100        assert_matches!(peek, Ok(PeekChar::Char(_)));
1101        lexer.consume_char();
1102
1103        let peek = lexer.peek_char().now_or_never().unwrap();
1104        assert_matches!(peek, Ok(PeekChar::Char(_)));
1105        lexer.consume_char();
1106    }
1107
1108    #[test]
1109    fn lexer_core_consume_char_success() {
1110        let input = Memory::new("a\nb");
1111        let line = NonZeroU64::new(1).unwrap();
1112        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1113
1114        let result = lexer.peek_char().now_or_never().unwrap();
1115        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1116            assert_eq!(c.value, 'a');
1117            assert_eq!(*c.location.code.value.borrow(), "a\n");
1118            assert_eq!(c.location.code.start_line_number, line);
1119            assert_eq!(*c.location.code.source, Source::Unknown);
1120            assert_eq!(c.location.range, 0..1);
1121        });
1122        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1123            assert_eq!(c.value, 'a');
1124            assert_eq!(*c.location.code.value.borrow(), "a\n");
1125            assert_eq!(c.location.code.start_line_number, line);
1126            assert_eq!(*c.location.code.source, Source::Unknown);
1127            assert_eq!(c.location.range, 0..1);
1128        });
1129        lexer.consume_char();
1130
1131        let result = lexer.peek_char().now_or_never().unwrap();
1132        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1133            assert_eq!(c.value, '\n');
1134            assert_eq!(*c.location.code.value.borrow(), "a\n");
1135            assert_eq!(c.location.code.start_line_number, line);
1136            assert_eq!(*c.location.code.source, Source::Unknown);
1137            assert_eq!(c.location.range, 1..2);
1138        });
1139        lexer.consume_char();
1140
1141        let result = lexer.peek_char().now_or_never().unwrap();
1142        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1143            assert_eq!(c.value, 'b');
1144            assert_eq!(*c.location.code.value.borrow(), "a\nb");
1145            assert_eq!(c.location.code.start_line_number.get(), 1);
1146            assert_eq!(*c.location.code.source, Source::Unknown);
1147            assert_eq!(c.location.range, 2..3);
1148        });
1149        lexer.consume_char();
1150
1151        let result = lexer.peek_char().now_or_never().unwrap();
1152        assert_matches!(result, Ok(PeekChar::EndOfInput(location)) => {
1153            assert_eq!(*location.code.value.borrow(), "a\nb");
1154            assert_eq!(location.code.start_line_number.get(), 1);
1155            assert_eq!(*location.code.source, Source::Unknown);
1156            assert_eq!(location.range, 3..3);
1157        });
1158    }
1159
1160    #[test]
1161    #[should_panic(expected = "A character must have been peeked before being consumed: index=0")]
1162    fn lexer_core_consume_char_panic() {
1163        let input = Memory::new("a");
1164        let line = NonZeroU64::new(1).unwrap();
1165        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1166        lexer.consume_char();
1167    }
1168
1169    #[test]
1170    fn lexer_core_peek_char_at() {
1171        let input = Memory::new("a\nb");
1172        let line = NonZeroU64::new(1).unwrap();
1173        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1174
1175        let c0 = assert_matches!(
1176            lexer.peek_char().now_or_never().unwrap(),
1177            Ok(PeekChar::Char(c)) => c.clone()
1178        );
1179        lexer.consume_char();
1180
1181        let c1 = assert_matches!(
1182            lexer.peek_char().now_or_never().unwrap(),
1183            Ok(PeekChar::Char(c)) => c.clone()
1184        );
1185        lexer.consume_char();
1186
1187        let c2 = assert_matches!(
1188            lexer.peek_char().now_or_never().unwrap(),
1189            Ok(PeekChar::Char(c)) => c.clone()
1190        );
1191
1192        assert_eq!(lexer.peek_char_at(0), &c0);
1193        assert_eq!(lexer.peek_char_at(1), &c1);
1194        assert_eq!(lexer.peek_char_at(2), &c2);
1195    }
1196
1197    #[test]
1198    fn lexer_core_index() {
1199        let input = Memory::new("a\nb");
1200        let line = NonZeroU64::new(1).unwrap();
1201        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1202
1203        assert_eq!(lexer.index(), 0);
1204        lexer.peek_char().now_or_never().unwrap().unwrap();
1205        assert_eq!(lexer.index(), 0);
1206        lexer.consume_char();
1207
1208        assert_eq!(lexer.index(), 1);
1209        lexer.peek_char().now_or_never().unwrap().unwrap();
1210        lexer.consume_char();
1211
1212        assert_eq!(lexer.index(), 2);
1213        lexer.peek_char().now_or_never().unwrap().unwrap();
1214        lexer.consume_char();
1215
1216        assert_eq!(lexer.index(), 3);
1217    }
1218
1219    #[test]
1220    fn lexer_core_rewind_success() {
1221        let input = Memory::new("abc");
1222        let line = NonZeroU64::new(1).unwrap();
1223        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1224        lexer.rewind(0);
1225        assert_eq!(lexer.index(), 0);
1226
1227        let _ = lexer.peek_char().now_or_never().unwrap();
1228        lexer.consume_char();
1229        let _ = lexer.peek_char().now_or_never().unwrap();
1230        lexer.consume_char();
1231        lexer.rewind(0);
1232
1233        let result = lexer.peek_char().now_or_never().unwrap();
1234        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1235            assert_eq!(c.value, 'a');
1236            assert_eq!(*c.location.code.value.borrow(), "abc");
1237            assert_eq!(c.location.code.start_line_number, line);
1238            assert_eq!(*c.location.code.source, Source::Unknown);
1239            assert_eq!(c.location.range, 0..1);
1240        });
1241    }
1242
1243    #[test]
1244    #[should_panic(expected = "The new index 1 must not be larger than the current index 0")]
1245    fn lexer_core_rewind_invalid_index() {
1246        let input = Memory::new("abc");
1247        let line = NonZeroU64::new(1).unwrap();
1248        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1249        lexer.rewind(1);
1250    }
1251
1252    #[test]
1253    fn lexer_core_source_string() {
1254        let input = Memory::new("ab\ncd");
1255        let line = NonZeroU64::new(1).unwrap();
1256        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1257        for _ in 0..4 {
1258            let _ = lexer.peek_char().now_or_never().unwrap();
1259            lexer.consume_char();
1260        }
1261
1262        let result = lexer.source_string(1..4);
1263        assert_eq!(result, "b\nc");
1264    }
1265
1266    #[test]
1267    #[should_panic(expected = "begin index 0 should be less than end index 0")]
1268    fn lexer_core_substitute_alias_with_invalid_index() {
1269        let input = Memory::new("a b");
1270        let line = NonZeroU64::new(1).unwrap();
1271        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1272        let alias = Rc::new(Alias {
1273            name: "a".to_string(),
1274            replacement: "".to_string(),
1275            global: false,
1276            origin: Location::dummy("dummy"),
1277        });
1278        lexer.substitute_alias(0, &alias);
1279    }
1280
1281    #[test]
1282    fn lexer_core_substitute_alias_single_line_replacement() {
1283        let input = Memory::new("a b");
1284        let line = NonZeroU64::new(1).unwrap();
1285        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1286        let alias = Rc::new(Alias {
1287            name: "a".to_string(),
1288            replacement: "lex".to_string(),
1289            global: false,
1290            origin: Location::dummy("dummy"),
1291        });
1292
1293        let _ = lexer.peek_char().now_or_never().unwrap();
1294        lexer.consume_char();
1295
1296        lexer.substitute_alias(0, &alias);
1297
1298        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1299            assert_eq!(c.value, 'l');
1300            assert_eq!(*c.location.code.value.borrow(), "lex");
1301            assert_eq!(c.location.code.start_line_number.get(), 1);
1302            assert_matches!(&*c.location.code.source,
1303                Source::Alias { original, alias: alias2 } => {
1304                assert_eq!(*original.code.value.borrow(), "a b");
1305                assert_eq!(original.code.start_line_number, line);
1306                assert_eq!(*original.code.source, Source::Unknown);
1307                assert_eq!(original.range, 0..1);
1308                assert_eq!(alias2, &alias);
1309            });
1310            assert_eq!(c.location.range, 0..1);
1311        });
1312        lexer.consume_char();
1313
1314        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1315            assert_eq!(c.value, 'e');
1316            assert_eq!(*c.location.code.value.borrow(), "lex");
1317            assert_eq!(c.location.code.start_line_number, line);
1318            assert_matches!(&*c.location.code.source,
1319                Source::Alias { original, alias: alias2 } => {
1320                assert_eq!(*original.code.value.borrow(), "a b");
1321                assert_eq!(original.code.start_line_number, line);
1322                assert_eq!(*original.code.source, Source::Unknown);
1323                assert_eq!(original.range, 0..1);
1324                assert_eq!(alias2, &alias);
1325            });
1326            assert_eq!(c.location.range, 1..2);
1327        });
1328        lexer.consume_char();
1329
1330        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1331            assert_eq!(c.value, 'x');
1332            assert_eq!(*c.location.code.value.borrow(), "lex");
1333            assert_eq!(c.location.code.start_line_number, line);
1334            assert_matches!(&*c.location.code.source,
1335                Source::Alias { original, alias: alias2 } => {
1336                assert_eq!(*original.code.value.borrow(), "a b");
1337                assert_eq!(original.code.start_line_number, line);
1338                assert_eq!(*original.code.source, Source::Unknown);
1339                assert_eq!(original.range, 0..1);
1340                assert_eq!(alias2, &alias);
1341            });
1342            assert_eq!(c.location.range, 2..3);
1343        });
1344        lexer.consume_char();
1345
1346        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1347            assert_eq!(c.value, ' ');
1348            assert_eq!(*c.location.code.value.borrow(), "a b");
1349            assert_eq!(c.location.code.start_line_number, line);
1350            assert_eq!(*c.location.code.source, Source::Unknown);
1351            assert_eq!(c.location.range, 1..2);
1352        });
1353        lexer.consume_char();
1354    }
1355
1356    #[test]
1357    fn lexer_core_substitute_alias_multi_line_replacement() {
1358        let input = Memory::new(" foo b");
1359        let line = NonZeroU64::new(1).unwrap();
1360        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1361        let alias = Rc::new(Alias {
1362            name: "foo".to_string(),
1363            replacement: "x\ny".to_string(),
1364            global: true,
1365            origin: Location::dummy("loc"),
1366        });
1367
1368        for _ in 0..4 {
1369            let _ = lexer.peek_char().now_or_never().unwrap();
1370            lexer.consume_char();
1371        }
1372
1373        lexer.substitute_alias(1, &alias);
1374
1375        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1376            assert_eq!(c.value, 'x');
1377            assert_eq!(*c.location.code.value.borrow(), "x\ny");
1378            assert_eq!(c.location.code.start_line_number, line);
1379            assert_matches!(&*c.location.code.source,
1380                Source::Alias { original, alias: alias2 } => {
1381                assert_eq!(*original.code.value.borrow(), " foo b");
1382                assert_eq!(original.code.start_line_number, line);
1383                assert_eq!(*original.code.source, Source::Unknown);
1384                assert_eq!(original.range, 1..4);
1385                assert_eq!(alias2, &alias);
1386            });
1387            assert_eq!(c.location.range, 0..1);
1388        });
1389        lexer.consume_char();
1390
1391        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1392            assert_eq!(c.value, '\n');
1393            assert_eq!(*c.location.code.value.borrow(), "x\ny");
1394            assert_eq!(c.location.code.start_line_number, line);
1395            assert_matches!(&*c.location.code.source,
1396                Source::Alias { original, alias: alias2 } => {
1397                assert_eq!(*original.code.value.borrow(), " foo b");
1398                assert_eq!(original.code.start_line_number, line);
1399                assert_eq!(*original.code.source, Source::Unknown);
1400                assert_eq!(original.range, 1..4);
1401                assert_eq!(alias2, &alias);
1402            });
1403            assert_eq!(c.location.range, 1..2);
1404        });
1405        lexer.consume_char();
1406
1407        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1408            assert_eq!(c.value, 'y');
1409            assert_eq!(*c.location.code.value.borrow(), "x\ny");
1410            assert_eq!(c.location.code.start_line_number, line);
1411            assert_matches!(&*c.location.code.source, Source::Alias { original, alias: alias2 } => {
1412                assert_eq!(*original.code.value.borrow(), " foo b");
1413                assert_eq!(original.code.start_line_number, line);
1414                assert_eq!(*original.code.source, Source::Unknown);
1415                assert_eq!(original.range, 1..4);
1416                assert_eq!(alias2, &alias);
1417            });
1418            assert_eq!(c.location.range, 2..3);
1419        });
1420        lexer.consume_char();
1421
1422        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1423            assert_eq!(c.value, ' ');
1424            assert_eq!(*c.location.code.value.borrow(), " foo b");
1425            assert_eq!(c.location.code.start_line_number, line);
1426            assert_eq!(*c.location.code.source, Source::Unknown);
1427            assert_eq!(c.location.range, 4..5);
1428        });
1429        lexer.consume_char();
1430    }
1431
1432    #[test]
1433    fn lexer_core_substitute_alias_empty_replacement() {
1434        let input = Memory::new("x ");
1435        let line = NonZeroU64::new(1).unwrap();
1436        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1437        let alias = Rc::new(Alias {
1438            name: "x".to_string(),
1439            replacement: "".to_string(),
1440            global: false,
1441            origin: Location::dummy("dummy"),
1442        });
1443
1444        let _ = lexer.peek_char().now_or_never().unwrap();
1445        lexer.consume_char();
1446
1447        lexer.substitute_alias(0, &alias);
1448
1449        assert_matches!(lexer.peek_char().now_or_never().unwrap(), Ok(PeekChar::Char(c)) => {
1450            assert_eq!(c.value, ' ');
1451            assert_eq!(*c.location.code.value.borrow(), "x ");
1452            assert_eq!(c.location.code.start_line_number, line);
1453            assert_eq!(*c.location.code.source, Source::Unknown);
1454            assert_eq!(c.location.range, 1..2);
1455        });
1456    }
1457
1458    #[test]
1459    fn lexer_core_peek_char_after_alias_substitution() {
1460        let input = Memory::new("a\nb");
1461        let line = NonZeroU64::new(1).unwrap();
1462        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1463
1464        lexer.peek_char().now_or_never().unwrap().unwrap();
1465        lexer.consume_char();
1466
1467        let alias = Rc::new(Alias {
1468            name: "a".to_string(),
1469            replacement: "".to_string(),
1470            global: false,
1471            origin: Location::dummy("dummy"),
1472        });
1473        lexer.substitute_alias(0, &alias);
1474
1475        let result = lexer.peek_char().now_or_never().unwrap();
1476        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1477            assert_eq!(c.value, '\n');
1478            assert_eq!(*c.location.code.value.borrow(), "a\n");
1479            assert_eq!(c.location.code.start_line_number, line);
1480            assert_eq!(*c.location.code.source, Source::Unknown);
1481            assert_eq!(c.location.range, 1..2);
1482        });
1483        lexer.consume_char();
1484
1485        let result = lexer.peek_char().now_or_never().unwrap();
1486        assert_matches!(result, Ok(PeekChar::Char(c)) => {
1487            assert_eq!(c.value, 'b');
1488            assert_eq!(*c.location.code.value.borrow(), "a\nb");
1489            assert_eq!(c.location.code.start_line_number.get(), 1);
1490            assert_eq!(*c.location.code.source, Source::Unknown);
1491            assert_eq!(c.location.range, 2..3);
1492        });
1493        lexer.consume_char();
1494
1495        let result = lexer.peek_char().now_or_never().unwrap();
1496        assert_matches!(result, Ok(PeekChar::EndOfInput(location)) => {
1497            assert_eq!(*location.code.value.borrow(), "a\nb");
1498            assert_eq!(location.code.start_line_number.get(), 1);
1499            assert_eq!(*location.code.source, Source::Unknown);
1500            assert_eq!(location.range, 3..3);
1501        });
1502    }
1503
1504    #[test]
1505    fn lexer_core_is_after_blank_ending_alias_index_0() {
1506        let original = Location::dummy("original");
1507        let alias = Rc::new(Alias {
1508            name: "a".to_string(),
1509            replacement: " ".to_string(),
1510            global: false,
1511            origin: Location::dummy("origin"),
1512        });
1513        let source = Source::Alias { original, alias };
1514        let input = Memory::new("a");
1515        let line = NonZeroU64::new(1).unwrap();
1516        let lexer = LexerCore::new(Box::new(input), line, Rc::new(source));
1517        assert!(!lexer.is_after_blank_ending_alias(0));
1518    }
1519
1520    #[test]
1521    fn lexer_core_is_after_blank_ending_alias_not_blank_ending() {
1522        let input = Memory::new("a x");
1523        let line = NonZeroU64::new(1).unwrap();
1524        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1525        let alias = Rc::new(Alias {
1526            name: "a".to_string(),
1527            replacement: " b".to_string(),
1528            global: false,
1529            origin: Location::dummy("dummy"),
1530        });
1531
1532        lexer.peek_char().now_or_never().unwrap().unwrap();
1533        lexer.consume_char();
1534
1535        lexer.substitute_alias(0, &alias);
1536
1537        assert!(!lexer.is_after_blank_ending_alias(0));
1538        assert!(!lexer.is_after_blank_ending_alias(1));
1539        assert!(!lexer.is_after_blank_ending_alias(2));
1540        assert!(!lexer.is_after_blank_ending_alias(3));
1541    }
1542
1543    #[test]
1544    fn lexer_core_is_after_blank_ending_alias_blank_ending() {
1545        let input = Memory::new("a x");
1546        let line = NonZeroU64::new(1).unwrap();
1547        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1548        let alias = Rc::new(Alias {
1549            name: "a".to_string(),
1550            replacement: " b ".to_string(),
1551            global: false,
1552            origin: Location::dummy("dummy"),
1553        });
1554
1555        lexer.peek_char().now_or_never().unwrap().unwrap();
1556        lexer.consume_char();
1557
1558        lexer.substitute_alias(0, &alias);
1559
1560        assert!(!lexer.is_after_blank_ending_alias(0));
1561        assert!(!lexer.is_after_blank_ending_alias(1));
1562        assert!(!lexer.is_after_blank_ending_alias(2));
1563        assert!(lexer.is_after_blank_ending_alias(3));
1564        assert!(lexer.is_after_blank_ending_alias(4));
1565    }
1566
1567    #[test]
1568    fn lexer_core_is_after_blank_ending_alias_after_line_continuation() {
1569        let input = Memory::new("a\\\n x");
1570        let line = NonZeroU64::new(1).unwrap();
1571        let mut lexer = LexerCore::new(Box::new(input), line, Rc::new(Source::Unknown));
1572        let alias = Rc::new(Alias {
1573            name: "a".to_string(),
1574            replacement: " b ".to_string(),
1575            global: false,
1576            origin: Location::dummy("dummy"),
1577        });
1578
1579        lexer.peek_char().now_or_never().unwrap().unwrap();
1580        lexer.consume_char();
1581        lexer.substitute_alias(0, &alias);
1582
1583        while let Ok(PeekChar::Char(_)) = lexer.peek_char().now_or_never().unwrap() {
1584            lexer.consume_char();
1585        }
1586        lexer.mark_line_continuation(3..5);
1587
1588        assert!(!lexer.is_after_blank_ending_alias(0));
1589        assert!(!lexer.is_after_blank_ending_alias(1));
1590        assert!(!lexer.is_after_blank_ending_alias(2));
1591        assert!(lexer.is_after_blank_ending_alias(5));
1592        assert!(lexer.is_after_blank_ending_alias(6));
1593    }
1594
1595    #[test]
1596    fn lexer_with_empty_source() {
1597        let mut lexer = Lexer::with_code("");
1598        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(None));
1599    }
1600
1601    #[test]
1602    fn lexer_peek_char_with_line_continuation_enabled_stopping_on_non_backslash() {
1603        let mut lexer = Lexer::with_code("\\\n\n\\");
1604        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n')));
1605        assert_eq!(lexer.index(), 2);
1606    }
1607
1608    #[test]
1609    fn lexer_peek_char_with_line_continuation_enabled_stopping_on_non_newline() {
1610        let mut lexer = Lexer::with_code("\\\n\\\n\\\n\\\\");
1611        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\\')));
1612        assert_eq!(lexer.index(), 6);
1613    }
1614
1615    #[test]
1616    fn lexer_peek_char_with_line_continuation_disabled() {
1617        let mut lexer = Lexer::with_code("\\\n\\\n\\\\");
1618        let mut lexer = lexer.disable_line_continuation();
1619        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\\')));
1620        assert_eq!(lexer.index(), 0);
1621    }
1622
1623    #[test]
1624    fn lexer_flush() {
1625        let mut lexer = Lexer::with_code(" \n\n\t\n");
1626        let location_1 = lexer.location().now_or_never().unwrap().unwrap().clone();
1627        assert_eq!(*location_1.code.value.borrow(), " \n");
1628
1629        lexer.consume_char();
1630        lexer.peek_char().now_or_never().unwrap().unwrap();
1631        lexer.consume_char();
1632        lexer.peek_char().now_or_never().unwrap().unwrap();
1633        lexer.consume_char();
1634        lexer.flush();
1635        lexer.peek_char().now_or_never().unwrap().unwrap();
1636        lexer.consume_char();
1637
1638        let location_2 = lexer.location().now_or_never().unwrap().unwrap().clone();
1639
1640        assert_eq!(*location_1.code.value.borrow(), " \n\n");
1641        assert_eq!(location_1.code.start_line_number.get(), 1);
1642        assert_eq!(*location_1.code.source, Source::Unknown);
1643        assert_eq!(location_1.range, 0..1);
1644        assert_eq!(*location_2.code.value.borrow(), "\t\n");
1645        assert_eq!(location_2.code.start_line_number.get(), 3);
1646        assert_eq!(*location_2.code.source, Source::Unknown);
1647        assert_eq!(location_2.range, 1..2);
1648    }
1649
1650    #[test]
1651    fn lexer_consume_char_if() {
1652        let mut lexer = Lexer::with_code("word\n");
1653
1654        let mut called = 0;
1655        let c = lexer
1656            .consume_char_if(|c| {
1657                assert_eq!(c, 'w');
1658                called += 1;
1659                true
1660            })
1661            .now_or_never()
1662            .unwrap()
1663            .unwrap()
1664            .unwrap();
1665        assert_eq!(called, 1);
1666        assert_eq!(c.value, 'w');
1667        assert_eq!(*c.location.code.value.borrow(), "word\n");
1668        assert_eq!(c.location.code.start_line_number.get(), 1);
1669        assert_eq!(*c.location.code.source, Source::Unknown);
1670        assert_eq!(c.location.range, 0..1);
1671
1672        let mut called = 0;
1673        let r = lexer
1674            .consume_char_if(|c| {
1675                assert_eq!(c, 'o');
1676                called += 1;
1677                false
1678            })
1679            .now_or_never()
1680            .unwrap();
1681        assert_eq!(called, 1);
1682        assert_eq!(r, Ok(None));
1683
1684        let mut called = 0;
1685        let r = lexer
1686            .consume_char_if(|c| {
1687                assert_eq!(c, 'o');
1688                called += 1;
1689                false
1690            })
1691            .now_or_never()
1692            .unwrap();
1693        assert_eq!(called, 1);
1694        assert_eq!(r, Ok(None));
1695
1696        let mut called = 0;
1697        let c = lexer
1698            .consume_char_if(|c| {
1699                assert_eq!(c, 'o');
1700                called += 1;
1701                true
1702            })
1703            .now_or_never()
1704            .unwrap()
1705            .unwrap()
1706            .unwrap();
1707        assert_eq!(called, 1);
1708        assert_eq!(c.value, 'o');
1709        assert_eq!(*c.location.code.value.borrow(), "word\n");
1710        assert_eq!(c.location.code.start_line_number.get(), 1);
1711        assert_eq!(*c.location.code.source, Source::Unknown);
1712        assert_eq!(c.location.range, 1..2);
1713
1714        lexer
1715            .consume_char_if(|c| {
1716                assert_eq!(c, 'r');
1717                true
1718            })
1719            .now_or_never()
1720            .unwrap()
1721            .unwrap()
1722            .unwrap();
1723        lexer
1724            .consume_char_if(|c| {
1725                assert_eq!(c, 'd');
1726                true
1727            })
1728            .now_or_never()
1729            .unwrap()
1730            .unwrap()
1731            .unwrap();
1732        lexer
1733            .consume_char_if(|c| {
1734                assert_eq!(c, '\n');
1735                true
1736            })
1737            .now_or_never()
1738            .unwrap()
1739            .unwrap()
1740            .unwrap();
1741
1742        // end of input
1743        let r = lexer
1744            .consume_char_if(|c| {
1745                unreachable!("unexpected call to the decider function: argument={}", c)
1746            })
1747            .now_or_never()
1748            .unwrap();
1749        assert_eq!(r, Ok(None));
1750    }
1751
1752    #[test]
1753    fn lexer_location_range_with_empty_range() {
1754        let mut lexer = Lexer::with_code("");
1755        lexer.peek_char().now_or_never().unwrap().unwrap();
1756        let location = lexer.location_range(0..0);
1757        assert_eq!(*location.code.value.borrow(), "");
1758        assert_eq!(location.code.start_line_number.get(), 1);
1759        assert_eq!(*location.code.source, Source::Unknown);
1760        assert_eq!(location.range, 0..0);
1761    }
1762
1763    #[test]
1764    fn lexer_location_range_with_nonempty_range() {
1765        let mut lexer = Lexer::from_memory("cat foo", Source::Stdin);
1766        for _ in 0..4 {
1767            lexer.peek_char().now_or_never().unwrap().unwrap();
1768            lexer.consume_char();
1769        }
1770        lexer.peek_char().now_or_never().unwrap().unwrap();
1771
1772        let location = lexer.location_range(1..4);
1773        assert_eq!(*location.code.value.borrow(), "cat foo");
1774        assert_eq!(location.code.start_line_number.get(), 1);
1775        assert_eq!(*location.code.source, Source::Stdin);
1776        assert_eq!(location.range, 1..4);
1777    }
1778
1779    #[test]
1780    fn lexer_location_range_with_range_starting_at_end() {
1781        let mut lexer = Lexer::from_memory("cat", Source::Stdin);
1782        for _ in 0..3 {
1783            lexer.peek_char().now_or_never().unwrap().unwrap();
1784            lexer.consume_char();
1785        }
1786        lexer.peek_char().now_or_never().unwrap().unwrap();
1787
1788        let location = lexer.location_range(3..3);
1789        assert_eq!(*location.code.value.borrow(), "cat");
1790        assert_eq!(location.code.start_line_number.get(), 1);
1791        assert_eq!(*location.code.source, Source::Stdin);
1792        assert_eq!(location.range, 3..3);
1793    }
1794
1795    #[test]
1796    #[should_panic]
1797    fn lexer_location_range_with_unconsumed_code() {
1798        let lexer = Lexer::with_code("echo ok");
1799        let _ = lexer.location_range(0..0);
1800    }
1801
1802    #[test]
1803    #[should_panic(expected = "The index 1 must not be larger than the current index 0")]
1804    fn lexer_location_range_with_range_out_of_bounds() {
1805        let lexer = Lexer::with_code("");
1806        let _ = lexer.location_range(1..2);
1807    }
1808
1809    #[test]
1810    fn lexer_location_range_with_alias_substitution() {
1811        let mut lexer = Lexer::with_code(" a;");
1812        let alias_def = Rc::new(Alias {
1813            name: "a".to_string(),
1814            replacement: "abc".to_string(),
1815            global: false,
1816            origin: Location::dummy("dummy"),
1817        });
1818        for _ in 0..2 {
1819            lexer.peek_char().now_or_never().unwrap().unwrap();
1820            lexer.consume_char();
1821        }
1822        lexer.substitute_alias(1, &alias_def);
1823        for _ in 1..5 {
1824            lexer.peek_char().now_or_never().unwrap().unwrap();
1825            lexer.consume_char();
1826        }
1827
1828        let location = lexer.location_range(2..5);
1829        assert_eq!(*location.code.value.borrow(), "abc");
1830        assert_eq!(location.code.start_line_number.get(), 1);
1831        assert_matches!(&*location.code.source, Source::Alias { original, alias } => {
1832            assert_eq!(*original.code.value.borrow(), " a;");
1833            assert_eq!(original.code.start_line_number.get(), 1);
1834            assert_eq!(*original.code.source, Source::Unknown);
1835            assert_eq!(original.range, 1..2);
1836            assert_eq!(alias, &alias_def);
1837        });
1838        assert_eq!(location.range, 1..3);
1839    }
1840
1841    #[test]
1842    fn lexer_inner_program_success() {
1843        let mut lexer = Lexer::with_code("x y )");
1844        let source = lexer.inner_program().now_or_never().unwrap().unwrap();
1845        assert_eq!(source, "x y ");
1846    }
1847
1848    #[test]
1849    fn lexer_inner_program_failure() {
1850        let mut lexer = Lexer::with_code("<< )");
1851        let e = lexer.inner_program().now_or_never().unwrap().unwrap_err();
1852        assert_eq!(
1853            e.cause,
1854            ErrorCause::Syntax(SyntaxError::MissingHereDocDelimiter)
1855        );
1856        assert_eq!(*e.location.code.value.borrow(), "<< )");
1857        assert_eq!(e.location.code.start_line_number.get(), 1);
1858        assert_eq!(*e.location.code.source, Source::Unknown);
1859        assert_eq!(e.location.range, 3..4);
1860    }
1861}
yash_syntax/parser/lex/core.rs

yash_syntax/parser/lex/
core.rs
