yash_syntax/parser/lex/
misc.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Extension of the core for implementing the rest of the lexer
18
19use super::core::Lexer;
20use super::core::is_blank;
21use crate::parser::core::Result;
22
23impl Lexer<'_> {
24    /// Skips a character if the given function returns true for it.
25    ///
26    /// Returns `Ok(true)` if the character was skipped, `Ok(false)` if the function returned
27    /// false, and `Err(_)` if an error occurred, respectively.
28    ///
29    /// `skip_if` is a simpler version of [`consume_char_if`](Lexer::consume_char_if).
30    pub async fn skip_if<F>(&mut self, f: F) -> Result<bool>
31    where
32        F: FnMut(char) -> bool,
33    {
34        Ok(self.consume_char_if(f).await?.is_some())
35    }
36
37    /// Skips blank characters until reaching a non-blank.
38    pub async fn skip_blanks(&mut self) -> Result<()> {
39        while self.skip_if(is_blank).await? {}
40        Ok(())
41    }
42
43    /// Skips a comment, if any.
44    ///
45    /// A comment ends just before a newline. The newline is *not* part of the comment.
46    ///
47    /// This function does not recognize line continuation inside the comment.
48    pub async fn skip_comment(&mut self) -> Result<()> {
49        if self.skip_if(|c| c == '#').await? {
50            let mut lexer = self.disable_line_continuation();
51            while lexer.skip_if(|c| c != '\n').await? {}
52            Lexer::enable_line_continuation(lexer);
53        }
54        Ok(())
55    }
56
57    /// Skips blank characters and a comment, if any.
58    ///
59    /// This function is the same as [`skip_blanks`](Lexer::skip_blanks)
60    /// followed by [`skip_comment`](Lexer::skip_comment).
61    pub async fn skip_blanks_and_comment(&mut self) -> Result<()> {
62        self.skip_blanks().await?;
63        self.skip_comment().await
64    }
65}
66
67#[cfg(test)]
68mod tests {
69    use super::*;
70    use futures_util::FutureExt;
71
72    #[test]
73    fn lexer_skip_blanks() {
74        let mut lexer = Lexer::with_code(" \t w");
75
76        let c = async {
77            lexer.skip_blanks().await?;
78            lexer.peek_char().await
79        }
80        .now_or_never()
81        .unwrap();
82        assert_eq!(c, Ok(Some('w')));
83
84        // Test idempotence
85        let c = async {
86            lexer.skip_blanks().await?;
87            lexer.peek_char().await
88        }
89        .now_or_never()
90        .unwrap();
91        assert_eq!(c, Ok(Some('w')));
92    }
93
94    #[test]
95    fn lexer_skip_blanks_does_not_skip_newline() {
96        let mut lexer = Lexer::with_code("\n");
97        lexer.skip_blanks().now_or_never().unwrap().unwrap();
98        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n')));
99    }
100
101    #[test]
102    fn lexer_skip_blanks_skips_line_continuations() {
103        let mut lexer = Lexer::with_code("\\\n  \\\n\\\n\\\n \\\nX");
104        let c = async {
105            lexer.skip_blanks().await?;
106            lexer.peek_char().await
107        }
108        .now_or_never()
109        .unwrap();
110        assert_eq!(c, Ok(Some('X')));
111
112        let mut lexer = Lexer::with_code("  \\\n\\\n  \\\n Y");
113        let c = async {
114            lexer.skip_blanks().await?;
115            lexer.peek_char().await
116        }
117        .now_or_never()
118        .unwrap();
119        assert_eq!(c, Ok(Some('Y')));
120    }
121
122    #[test]
123    fn lexer_skip_comment_no_comment() {
124        let mut lexer = Lexer::with_code("\n");
125        lexer.skip_comment().now_or_never().unwrap().unwrap();
126        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n')));
127    }
128
129    #[test]
130    fn lexer_skip_comment_empty_comment() {
131        let mut lexer = Lexer::with_code("#\n");
132
133        let c = async {
134            lexer.skip_comment().await?;
135            lexer.peek_char().await
136        }
137        .now_or_never()
138        .unwrap();
139        assert_eq!(c, Ok(Some('\n')));
140
141        // Test idempotence
142        let c = async {
143            lexer.skip_comment().await?;
144            lexer.peek_char().await
145        }
146        .now_or_never()
147        .unwrap();
148        assert_eq!(c, Ok(Some('\n')));
149    }
150
151    #[test]
152    fn lexer_skip_comment_non_empty_comment() {
153        let mut lexer = Lexer::with_code("\\\n### foo bar\\\n");
154
155        let c = async {
156            lexer.skip_comment().await?;
157            lexer.peek_char().await
158        }
159        .now_or_never()
160        .unwrap();
161        assert_eq!(c, Ok(Some('\n')));
162        assert_eq!(lexer.index(), 14);
163
164        // Test idempotence
165        let c = async {
166            lexer.skip_comment().await?;
167            lexer.peek_char().await
168        }
169        .now_or_never()
170        .unwrap();
171        assert_eq!(c, Ok(Some('\n')));
172        assert_eq!(lexer.index(), 14);
173    }
174
175    #[test]
176    fn lexer_skip_comment_not_ending_with_newline() {
177        let mut lexer = Lexer::with_code("#comment");
178
179        let c = async {
180            lexer.skip_comment().await?;
181            lexer.peek_char().await
182        }
183        .now_or_never()
184        .unwrap();
185        assert_eq!(c, Ok(None));
186
187        // Test idempotence
188        let c = async {
189            lexer.skip_comment().await?;
190            lexer.peek_char().await
191        }
192        .now_or_never()
193        .unwrap();
194        assert_eq!(c, Ok(None));
195    }
196}
yash_syntax/parser/lex/misc.rs

yash_syntax/parser/lex/
misc.rs
