yash_syntax/parser/lex/
misc.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2020 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Extension of the core for implementing the rest of the lexer.
18
19use super::core::is_blank;
20use super::core::Lexer;
21use crate::parser::core::Result;
22
23impl Lexer<'_> {
24    /// Skips a character if the given function returns true for it.
25    ///
26    /// Returns `Ok(true)` if the character was skipped, `Ok(false)` if the function returned
27    /// false, and `Err(_)` if an error occurred, respectively.
28    ///
29    /// `skip_if` is a simpler version of [`consume_char_if`](Lexer::consume_char_if).
30    pub async fn skip_if<F>(&mut self, f: F) -> Result<bool>
31    where
32        F: FnMut(char) -> bool,
33    {
34        Ok(self.consume_char_if(f).await?.is_some())
35    }
36
37    /// Skips blank characters until reaching a non-blank.
38    pub async fn skip_blanks(&mut self) -> Result<()> {
39        while self.skip_if(is_blank).await? {}
40        Ok(())
41    }
42
43    /// Skips a comment, if any.
44    ///
45    /// A comment ends just before a newline. The newline is *not* part of the comment.
46    ///
47    /// This function does not recognize line continuation inside the comment.
48    pub async fn skip_comment(&mut self) -> Result<()> {
49        if self.skip_if(|c| c == '#').await? {
50            let mut lexer = self.disable_line_continuation();
51            while lexer.skip_if(|c| c != '\n').await? {}
52            Lexer::enable_line_continuation(lexer);
53        }
54        Ok(())
55    }
56
57    /// Skips blank characters and a comment, if any.
58    ///
59    /// This function is the same as [`skip_blanks`](Lexer::skip_blanks)
60    /// followed by [`skip_comment`](Lexer::skip_comment).
61    pub async fn skip_blanks_and_comment(&mut self) -> Result<()> {
62        self.skip_blanks().await?;
63        self.skip_comment().await
64    }
65}
66
67#[cfg(test)]
68mod tests {
69    use super::*;
70    use crate::source::Source;
71    use futures_util::FutureExt;
72
73    #[test]
74    fn lexer_skip_blanks() {
75        let mut lexer = Lexer::from_memory(" \t w", Source::Unknown);
76
77        let c = async {
78            lexer.skip_blanks().await?;
79            lexer.peek_char().await
80        }
81        .now_or_never()
82        .unwrap();
83        assert_eq!(c, Ok(Some('w')));
84
85        // Test idempotence
86        let c = async {
87            lexer.skip_blanks().await?;
88            lexer.peek_char().await
89        }
90        .now_or_never()
91        .unwrap();
92        assert_eq!(c, Ok(Some('w')));
93    }
94
95    #[test]
96    fn lexer_skip_blanks_does_not_skip_newline() {
97        let mut lexer = Lexer::from_memory("\n", Source::Unknown);
98        lexer.skip_blanks().now_or_never().unwrap().unwrap();
99        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n')));
100    }
101
102    #[test]
103    fn lexer_skip_blanks_skips_line_continuations() {
104        let mut lexer = Lexer::from_memory("\\\n  \\\n\\\n\\\n \\\nX", Source::Unknown);
105        let c = async {
106            lexer.skip_blanks().await?;
107            lexer.peek_char().await
108        }
109        .now_or_never()
110        .unwrap();
111        assert_eq!(c, Ok(Some('X')));
112
113        let mut lexer = Lexer::from_memory("  \\\n\\\n  \\\n Y", Source::Unknown);
114        let c = async {
115            lexer.skip_blanks().await?;
116            lexer.peek_char().await
117        }
118        .now_or_never()
119        .unwrap();
120        assert_eq!(c, Ok(Some('Y')));
121    }
122
123    #[test]
124    fn lexer_skip_comment_no_comment() {
125        let mut lexer = Lexer::from_memory("\n", Source::Unknown);
126        lexer.skip_comment().now_or_never().unwrap().unwrap();
127        assert_eq!(lexer.peek_char().now_or_never().unwrap(), Ok(Some('\n')));
128    }
129
130    #[test]
131    fn lexer_skip_comment_empty_comment() {
132        let mut lexer = Lexer::from_memory("#\n", Source::Unknown);
133
134        let c = async {
135            lexer.skip_comment().await?;
136            lexer.peek_char().await
137        }
138        .now_or_never()
139        .unwrap();
140        assert_eq!(c, Ok(Some('\n')));
141
142        // Test idempotence
143        let c = async {
144            lexer.skip_comment().await?;
145            lexer.peek_char().await
146        }
147        .now_or_never()
148        .unwrap();
149        assert_eq!(c, Ok(Some('\n')));
150    }
151
152    #[test]
153    fn lexer_skip_comment_non_empty_comment() {
154        let mut lexer = Lexer::from_memory("\\\n### foo bar\\\n", Source::Unknown);
155
156        let c = async {
157            lexer.skip_comment().await?;
158            lexer.peek_char().await
159        }
160        .now_or_never()
161        .unwrap();
162        assert_eq!(c, Ok(Some('\n')));
163        assert_eq!(lexer.index(), 14);
164
165        // Test idempotence
166        let c = async {
167            lexer.skip_comment().await?;
168            lexer.peek_char().await
169        }
170        .now_or_never()
171        .unwrap();
172        assert_eq!(c, Ok(Some('\n')));
173        assert_eq!(lexer.index(), 14);
174    }
175
176    #[test]
177    fn lexer_skip_comment_not_ending_with_newline() {
178        let mut lexer = Lexer::from_memory("#comment", Source::Unknown);
179
180        let c = async {
181            lexer.skip_comment().await?;
182            lexer.peek_char().await
183        }
184        .now_or_never()
185        .unwrap();
186        assert_eq!(c, Ok(None));
187
188        // Test idempotence
189        let c = async {
190            lexer.skip_comment().await?;
191            lexer.peek_char().await
192        }
193        .now_or_never()
194        .unwrap();
195        assert_eq!(c, Ok(None));
196    }
197}
yash_syntax/parser/lex/misc.rs

yash_syntax/parser/lex/
misc.rs
