yash_arith/
token.rs

1// This file is part of yash, an extended POSIX shell.
2// Copyright (C) 2022 WATANABE Yuki
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12// GNU General Public License for more details.
13//
14// You should have received a copy of the GNU General Public License
15// along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17//! Tokenization
18
19use std::fmt::Display;
20use std::iter::FusedIterator;
21use std::ops::Range;
22use thiserror::Error;
23
24/// Result of evaluating an expression
25///
26/// TODO: The current implementation only supports integer arithmetic. A future
27/// version may also support floating-point numbers.
28#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
29pub enum Value {
30    Integer(i64),
31}
32
33impl Display for Value {
34    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
35        match self {
36            Value::Integer(i) => i.fmt(f),
37        }
38    }
39}
40
41/// Intermediate result of evaluating part of an expression
42#[derive(Clone, Debug, Eq, Hash, PartialEq)]
43pub enum Term<'a> {
44    /// Value
45    Value(Value),
46    /// Variable
47    Variable {
48        /// Variable name
49        name: &'a str,
50        /// Range of the substring in the evaluated expression where the variable occurs
51        location: Range<usize>,
52    },
53}
54
55/// Operator
56#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
57pub enum Operator {
58    /// `?`
59    Question,
60    /// `:`
61    Colon,
62    /// `|`
63    Bar,
64    /// `||`
65    BarBar,
66    /// `|=`
67    BarEqual,
68    /// `^`
69    Caret,
70    /// `^=`
71    CaretEqual,
72    /// `&`
73    And,
74    /// `&&`
75    AndAnd,
76    /// `&=`
77    AndEqual,
78    /// `=`
79    Equal,
80    /// `==`
81    EqualEqual,
82    /// `!`
83    Bang,
84    /// `!=`
85    BangEqual,
86    /// `<`
87    Less,
88    /// `<=`
89    LessEqual,
90    /// `<<`
91    LessLess,
92    /// `<<=`
93    LessLessEqual,
94    /// `>`
95    Greater,
96    /// `>=`
97    GreaterEqual,
98    /// `>>`
99    GreaterGreater,
100    /// `>>=`
101    GreaterGreaterEqual,
102    /// `+`
103    Plus,
104    /// `++`
105    PlusPlus,
106    /// `+=`
107    PlusEqual,
108    /// `-`
109    Minus,
110    /// `--`
111    MinusMinus,
112    /// `-=`
113    MinusEqual,
114    /// `*`
115    Asterisk,
116    /// `*=`
117    AsteriskEqual,
118    /// `/`
119    Slash,
120    /// `/=`
121    SlashEqual,
122    /// `%`
123    Percent,
124    /// `%=`
125    PercentEqual,
126    /// `~`
127    Tilde,
128    /// `(`
129    OpenParen,
130    /// `)`
131    CloseParen,
132}
133
134/// Value of a [`Token`].
135#[derive(Clone, Debug, Eq, Hash, PartialEq)]
136pub enum TokenValue<'a> {
137    /// Term
138    Term(Term<'a>),
139    /// Operator
140    Operator(Operator),
141    /// Imaginary token value for the end of input.
142    EndOfInput,
143}
144
145/// Atomic lexical element of an expression
146#[derive(Clone, Debug, Eq, Hash, PartialEq)]
147pub struct Token<'a> {
148    /// Token value
149    pub value: TokenValue<'a>,
150    /// Range of the substring where the token occurs in the parsed expression
151    pub location: Range<usize>,
152}
153
154/// Cause of a tokenization error
155#[derive(Clone, Debug, Eq, Error, Hash, PartialEq)]
156pub enum TokenError {
157    /// A value token contains an invalid character.
158    #[error("invalid numeric constant")]
159    InvalidNumericConstant,
160
161    /// An expression contains a character that is not a whitespace, operator,
162    /// or number.
163    #[error("invalid character")]
164    InvalidCharacter,
165}
166
167/// Description of an error that occurred during expansion
168#[derive(Clone, Debug, Eq, Hash, PartialEq)]
169pub struct Error {
170    /// Cause of the error
171    pub cause: TokenError,
172    /// Range of the substring in the evaluated expression string where the error occurred
173    pub location: Range<usize>,
174}
175
176/// List of all the operators.
177///
178/// If a prefix of a valid operator is another operator, the prefix (the shorter
179/// operator) must appear after the longer. With this ordering, we can
180/// short-circuit unnecessary matching on finding a first match.
181const OPERATORS: &[(&str, Operator)] = &[
182    ("?", Operator::Question),
183    (":", Operator::Colon),
184    ("|=", Operator::BarEqual),
185    ("||", Operator::BarBar),
186    ("|", Operator::Bar),
187    ("^=", Operator::CaretEqual),
188    ("^", Operator::Caret),
189    ("&=", Operator::AndEqual),
190    ("&&", Operator::AndAnd),
191    ("&", Operator::And),
192    ("==", Operator::EqualEqual),
193    ("=", Operator::Equal),
194    ("!=", Operator::BangEqual),
195    ("<=", Operator::LessEqual),
196    ("<<=", Operator::LessLessEqual),
197    ("<<", Operator::LessLess),
198    ("<", Operator::Less),
199    (">=", Operator::GreaterEqual),
200    (">>=", Operator::GreaterGreaterEqual),
201    (">>", Operator::GreaterGreater),
202    (">", Operator::Greater),
203    ("+=", Operator::PlusEqual),
204    ("++", Operator::PlusPlus),
205    ("+", Operator::Plus),
206    ("-=", Operator::MinusEqual),
207    ("--", Operator::MinusMinus),
208    ("-", Operator::Minus),
209    ("*=", Operator::AsteriskEqual),
210    ("*", Operator::Asterisk),
211    ("/=", Operator::SlashEqual),
212    ("/", Operator::Slash),
213    ("%=", Operator::PercentEqual),
214    ("%", Operator::Percent),
215    ("~", Operator::Tilde),
216    ("!", Operator::Bang),
217    ("(", Operator::OpenParen),
218    (")", Operator::CloseParen),
219];
220
221/// Iterator extracting tokens from a string
222///
223/// `Tokens` implements `Iterator` but never yields `None` because it returns a
224/// special token with `TokenValue::EndOfInput` when there are no more tokens.
225/// The `next_token` inherent method may be handier than the methods of
226/// `Iterator` since it returns tokens without wrapping them in `Option`.
227///
228/// See also [`PeekableTokens`], which makes the iterator peekable.
229#[derive(Clone, Debug, Eq, Hash, PartialEq)]
230pub struct Tokens<'a> {
231    source: &'a str,
232    index: usize,
233}
234
235impl<'a> Tokens<'a> {
236    /// Creates a tokenizer.
237    pub fn new(source: &'a str) -> Self {
238        Tokens { source, index: 0 }
239    }
240
241    pub fn next_token(&mut self) -> Result<Token<'a>, Error> {
242        let source = self.source[self.index..].trim_start();
243        let start_of_token = self.source.len() - source.len();
244        let first_char = if let Some(c) = source.chars().next() {
245            c
246        } else {
247            return Ok(Token {
248                value: TokenValue::EndOfInput,
249                location: start_of_token..start_of_token,
250            });
251        };
252
253        if let Some((lexeme, operator)) = OPERATORS
254            .iter()
255            .copied()
256            .find(|&(lexeme, _)| source.starts_with(lexeme))
257        {
258            // Okay, this is an operator.
259            let end_of_token = start_of_token + lexeme.len();
260            let location = start_of_token..end_of_token;
261            self.index = end_of_token;
262            Ok(Token {
263                value: TokenValue::Operator(operator),
264                location,
265            })
266        } else {
267            // The next token should be a term. Try parsing it.
268            let remainder = source.trim_start_matches(|c: char| c.is_alphanumeric() || c == '_');
269            let token_len = source.len() - remainder.len();
270            if token_len == 0 {
271                return Err(Error {
272                    cause: TokenError::InvalidCharacter,
273                    location: start_of_token..start_of_token + 1,
274                });
275            }
276            let end_of_token = start_of_token + token_len;
277            let location = start_of_token..end_of_token;
278            let token = &source[..token_len];
279            let term = if first_char.is_ascii_digit() {
280                let parse = if let Some(token_source) = token.strip_prefix("0X") {
281                    i64::from_str_radix(token_source, 0x10)
282                } else if let Some(token_source) = token.strip_prefix("0x") {
283                    i64::from_str_radix(token_source, 0x10)
284                } else if source.starts_with('0') {
285                    i64::from_str_radix(token, 0o10)
286                } else {
287                    token.parse()
288                };
289                match parse {
290                    Ok(i) => Term::Value(Value::Integer(i)),
291                    Err(_) => {
292                        return Err(Error {
293                            cause: TokenError::InvalidNumericConstant,
294                            location,
295                        })
296                    }
297                }
298            } else {
299                Term::Variable {
300                    name: token,
301                    location: location.clone(),
302                }
303            };
304
305            self.index = end_of_token;
306            Ok(Token {
307                value: TokenValue::Term(term),
308                location,
309            })
310        }
311    }
312}
313
314impl<'a> Iterator for Tokens<'a> {
315    type Item = Result<Token<'a>, Error>;
316
317    fn next(&mut self) -> Option<Result<Token<'a>, Error>> {
318        Some(self.next_token())
319    }
320}
321
322/// `Tokens` is fused because it never yields `None`.
323impl FusedIterator for Tokens<'_> {}
324
325/// Peekable iterator extracting tokens from a string
326///
327/// `PeekableTokens` works as a wrapper of [`Tokens`] that adds the
328/// [`peek`](Self::peek) method.
329#[derive(Clone, Debug, Eq, Hash, PartialEq)]
330pub struct PeekableTokens<'a> {
331    inner: Tokens<'a>,
332    cached_next: Option<Result<Token<'a>, Error>>,
333}
334
335impl<'a> PeekableTokens<'a> {
336    /// Creates a tokenizer.
337    pub fn new(inner: Tokens<'a>) -> Self {
338        let cached_next = None;
339        PeekableTokens { inner, cached_next }
340    }
341
342    /// Consumes and returns the next token.
343    pub fn next(&mut self) -> Result<Token<'a>, Error> {
344        self.cached_next
345            .take()
346            .unwrap_or_else(|| self.inner.next_token())
347    }
348
349    /// Returns the next token without consuming it.
350    ///
351    /// The token will be returned again on a next call to `peek` or
352    /// [`next`](Self::next).
353    pub fn peek(&mut self) -> &Result<Token<'a>, Error> {
354        self.cached_next
355            .get_or_insert_with(|| self.inner.next_token())
356    }
357}
358
359impl<'a> From<&'a str> for PeekableTokens<'a> {
360    fn from(source: &'a str) -> Self {
361        PeekableTokens::new(Tokens::new(source))
362    }
363}
364
365#[cfg(test)]
366mod tests {
367    use super::*;
368
369    #[test]
370    fn decimal_integer_constants() {
371        assert_eq!(
372            Tokens::new("1").next(),
373            Some(Ok(Token {
374                value: TokenValue::Term(Term::Value(Value::Integer(1))),
375                location: 0..1,
376            }))
377        );
378        assert_eq!(
379            Tokens::new("42").next(),
380            Some(Ok(Token {
381                value: TokenValue::Term(Term::Value(Value::Integer(42))),
382                location: 0..2,
383            }))
384        );
385    }
386
387    #[test]
388    fn invalid_digit_in_decimal_constant() {
389        assert_eq!(
390            Tokens::new("1a").next(),
391            Some(Err(Error {
392                cause: TokenError::InvalidNumericConstant,
393                location: 0..2,
394            }))
395        );
396        assert_eq!(
397            Tokens::new("  123_456 ").next(),
398            Some(Err(Error {
399                cause: TokenError::InvalidNumericConstant,
400                location: 2..9,
401            }))
402        );
403    }
404
405    #[test]
406    fn octal_integer_constants() {
407        assert_eq!(
408            Tokens::new("0").next(),
409            Some(Ok(Token {
410                value: TokenValue::Term(Term::Value(Value::Integer(0))),
411                location: 0..1,
412            }))
413        );
414        assert_eq!(
415            Tokens::new("01").next(),
416            Some(Ok(Token {
417                value: TokenValue::Term(Term::Value(Value::Integer(0o1))),
418                location: 0..2,
419            }))
420        );
421        assert_eq!(
422            Tokens::new("07").next(),
423            Some(Ok(Token {
424                value: TokenValue::Term(Term::Value(Value::Integer(0o7))),
425                location: 0..2,
426            }))
427        );
428        assert_eq!(
429            Tokens::new("0123").next(),
430            Some(Ok(Token {
431                value: TokenValue::Term(Term::Value(Value::Integer(0o123))),
432                location: 0..4,
433            }))
434        );
435    }
436
437    #[test]
438    fn invalid_digit_in_octal_constant() {
439        assert_eq!(
440            Tokens::new("08").next(),
441            Some(Err(Error {
442                cause: TokenError::InvalidNumericConstant,
443                location: 0..2,
444            }))
445        );
446        assert_eq!(
447            Tokens::new(" 0192 ").next(),
448            Some(Err(Error {
449                cause: TokenError::InvalidNumericConstant,
450                location: 1..5,
451            }))
452        );
453        assert_eq!(
454            Tokens::new("0ab").next(),
455            Some(Err(Error {
456                cause: TokenError::InvalidNumericConstant,
457                location: 0..3,
458            }))
459        );
460    }
461
462    #[test]
463    fn hexadecimal_integer_constants() {
464        assert_eq!(
465            Tokens::new("0x0").next(),
466            Some(Ok(Token {
467                value: TokenValue::Term(Term::Value(Value::Integer(0x0))),
468                location: 0..3,
469            }))
470        );
471        assert_eq!(
472            Tokens::new("0X1").next(),
473            Some(Ok(Token {
474                value: TokenValue::Term(Term::Value(Value::Integer(0x1))),
475                location: 0..3,
476            }))
477        );
478        assert_eq!(
479            Tokens::new("0x19Af").next(),
480            Some(Ok(Token {
481                value: TokenValue::Term(Term::Value(Value::Integer(0x19AF))),
482                location: 0..6,
483            }))
484        );
485    }
486
487    #[test]
488    fn broken_hexadecimal_integer_constants() {
489        assert_eq!(
490            Tokens::new("0x").next(),
491            Some(Err(Error {
492                cause: TokenError::InvalidNumericConstant,
493                location: 0..2,
494            }))
495        );
496        assert_eq!(
497            Tokens::new(" 0xG ").next(),
498            Some(Err(Error {
499                cause: TokenError::InvalidNumericConstant,
500                location: 1..4,
501            }))
502        );
503        assert_eq!(
504            Tokens::new("0x1z2").next(),
505            Some(Err(Error {
506                cause: TokenError::InvalidNumericConstant,
507                location: 0..5,
508            }))
509        );
510    }
511
512    // TODO Float constants
513
514    #[test]
515    fn variables() {
516        assert_eq!(
517            Tokens::new("abc").next(),
518            Some(Ok(Token {
519                value: TokenValue::Term(Term::Variable {
520                    name: "abc",
521                    location: 0..3,
522                }),
523                location: 0..3,
524            }))
525        );
526        assert_eq!(
527            Tokens::new("foo_BAR").next(),
528            Some(Ok(Token {
529                value: TokenValue::Term(Term::Variable {
530                    name: "foo_BAR",
531                    location: 0..7,
532                }),
533                location: 0..7,
534            }))
535        );
536        assert_eq!(
537            Tokens::new("a1B2c").next(),
538            Some(Ok(Token {
539                value: TokenValue::Term(Term::Variable {
540                    name: "a1B2c",
541                    location: 0..5,
542                }),
543                location: 0..5,
544            }))
545        );
546        assert_eq!(
547            Tokens::new(" _var").next(),
548            Some(Ok(Token {
549                value: TokenValue::Term(Term::Variable {
550                    name: "_var",
551                    location: 1..5,
552                }),
553                location: 1..5,
554            }))
555        );
556    }
557
558    #[test]
559    fn operators() {
560        assert_eq!(
561            Tokens::new("?").next(),
562            Some(Ok(Token {
563                value: TokenValue::Operator(Operator::Question),
564                location: 0..1,
565            }))
566        );
567        assert_eq!(
568            Tokens::new(":").next(),
569            Some(Ok(Token {
570                value: TokenValue::Operator(Operator::Colon),
571                location: 0..1,
572            }))
573        );
574        assert_eq!(
575            Tokens::new("|").next(),
576            Some(Ok(Token {
577                value: TokenValue::Operator(Operator::Bar),
578                location: 0..1,
579            }))
580        );
581        assert_eq!(
582            Tokens::new("||").next(),
583            Some(Ok(Token {
584                value: TokenValue::Operator(Operator::BarBar),
585                location: 0..2,
586            }))
587        );
588        assert_eq!(
589            Tokens::new("|=").next(),
590            Some(Ok(Token {
591                value: TokenValue::Operator(Operator::BarEqual),
592                location: 0..2,
593            }))
594        );
595        assert_eq!(
596            Tokens::new("^").next(),
597            Some(Ok(Token {
598                value: TokenValue::Operator(Operator::Caret),
599                location: 0..1,
600            }))
601        );
602        assert_eq!(
603            Tokens::new("^=").next(),
604            Some(Ok(Token {
605                value: TokenValue::Operator(Operator::CaretEqual),
606                location: 0..2,
607            }))
608        );
609        assert_eq!(
610            Tokens::new("&").next(),
611            Some(Ok(Token {
612                value: TokenValue::Operator(Operator::And),
613                location: 0..1,
614            }))
615        );
616        assert_eq!(
617            Tokens::new("&&").next(),
618            Some(Ok(Token {
619                value: TokenValue::Operator(Operator::AndAnd),
620                location: 0..2,
621            }))
622        );
623        assert_eq!(
624            Tokens::new("&=").next(),
625            Some(Ok(Token {
626                value: TokenValue::Operator(Operator::AndEqual),
627                location: 0..2,
628            }))
629        );
630        assert_eq!(
631            Tokens::new("=").next(),
632            Some(Ok(Token {
633                value: TokenValue::Operator(Operator::Equal),
634                location: 0..1,
635            }))
636        );
637        assert_eq!(
638            Tokens::new("==").next(),
639            Some(Ok(Token {
640                value: TokenValue::Operator(Operator::EqualEqual),
641                location: 0..2,
642            }))
643        );
644        assert_eq!(
645            Tokens::new("!=").next(),
646            Some(Ok(Token {
647                value: TokenValue::Operator(Operator::BangEqual),
648                location: 0..2,
649            }))
650        );
651        assert_eq!(
652            Tokens::new("<").next(),
653            Some(Ok(Token {
654                value: TokenValue::Operator(Operator::Less),
655                location: 0..1,
656            }))
657        );
658        assert_eq!(
659            Tokens::new("<=").next(),
660            Some(Ok(Token {
661                value: TokenValue::Operator(Operator::LessEqual),
662                location: 0..2,
663            }))
664        );
665        assert_eq!(
666            Tokens::new("<<").next(),
667            Some(Ok(Token {
668                value: TokenValue::Operator(Operator::LessLess),
669                location: 0..2,
670            }))
671        );
672        assert_eq!(
673            Tokens::new("<<=").next(),
674            Some(Ok(Token {
675                value: TokenValue::Operator(Operator::LessLessEqual),
676                location: 0..3,
677            }))
678        );
679        assert_eq!(
680            Tokens::new(">").next(),
681            Some(Ok(Token {
682                value: TokenValue::Operator(Operator::Greater),
683                location: 0..1,
684            }))
685        );
686        assert_eq!(
687            Tokens::new(">=").next(),
688            Some(Ok(Token {
689                value: TokenValue::Operator(Operator::GreaterEqual),
690                location: 0..2,
691            }))
692        );
693        assert_eq!(
694            Tokens::new(">>").next(),
695            Some(Ok(Token {
696                value: TokenValue::Operator(Operator::GreaterGreater),
697                location: 0..2,
698            }))
699        );
700        assert_eq!(
701            Tokens::new(">>=").next(),
702            Some(Ok(Token {
703                value: TokenValue::Operator(Operator::GreaterGreaterEqual),
704                location: 0..3,
705            }))
706        );
707        assert_eq!(
708            Tokens::new("+").next(),
709            Some(Ok(Token {
710                value: TokenValue::Operator(Operator::Plus),
711                location: 0..1,
712            }))
713        );
714        assert_eq!(
715            Tokens::new("++").next(),
716            Some(Ok(Token {
717                value: TokenValue::Operator(Operator::PlusPlus),
718                location: 0..2,
719            }))
720        );
721        assert_eq!(
722            Tokens::new("+=").next(),
723            Some(Ok(Token {
724                value: TokenValue::Operator(Operator::PlusEqual),
725                location: 0..2,
726            }))
727        );
728        assert_eq!(
729            Tokens::new("-").next(),
730            Some(Ok(Token {
731                value: TokenValue::Operator(Operator::Minus),
732                location: 0..1,
733            }))
734        );
735        assert_eq!(
736            Tokens::new("--").next(),
737            Some(Ok(Token {
738                value: TokenValue::Operator(Operator::MinusMinus),
739                location: 0..2,
740            }))
741        );
742        assert_eq!(
743            Tokens::new("-=").next(),
744            Some(Ok(Token {
745                value: TokenValue::Operator(Operator::MinusEqual),
746                location: 0..2,
747            }))
748        );
749        assert_eq!(
750            Tokens::new("*").next(),
751            Some(Ok(Token {
752                value: TokenValue::Operator(Operator::Asterisk),
753                location: 0..1,
754            }))
755        );
756        assert_eq!(
757            Tokens::new("*=").next(),
758            Some(Ok(Token {
759                value: TokenValue::Operator(Operator::AsteriskEqual),
760                location: 0..2,
761            }))
762        );
763        assert_eq!(
764            Tokens::new("/").next(),
765            Some(Ok(Token {
766                value: TokenValue::Operator(Operator::Slash),
767                location: 0..1,
768            }))
769        );
770        assert_eq!(
771            Tokens::new("/=").next(),
772            Some(Ok(Token {
773                value: TokenValue::Operator(Operator::SlashEqual),
774                location: 0..2,
775            }))
776        );
777        assert_eq!(
778            Tokens::new("%").next(),
779            Some(Ok(Token {
780                value: TokenValue::Operator(Operator::Percent),
781                location: 0..1,
782            }))
783        );
784        assert_eq!(
785            Tokens::new("%=").next(),
786            Some(Ok(Token {
787                value: TokenValue::Operator(Operator::PercentEqual),
788                location: 0..2,
789            }))
790        );
791        assert_eq!(
792            Tokens::new("~").next(),
793            Some(Ok(Token {
794                value: TokenValue::Operator(Operator::Tilde),
795                location: 0..1,
796            }))
797        );
798        assert_eq!(
799            Tokens::new("!").next(),
800            Some(Ok(Token {
801                value: TokenValue::Operator(Operator::Bang),
802                location: 0..1,
803            }))
804        );
805        assert_eq!(
806            Tokens::new("(").next(),
807            Some(Ok(Token {
808                value: TokenValue::Operator(Operator::OpenParen),
809                location: 0..1
810            }))
811        );
812        assert_eq!(
813            Tokens::new("(").next(),
814            Some(Ok(Token {
815                value: TokenValue::Operator(Operator::OpenParen),
816                location: 0..1
817            }))
818        );
819    }
820
821    #[test]
822    fn space_around_token() {
823        assert_eq!(
824            Tokens::new(" 42").next(),
825            Some(Ok(Token {
826                value: TokenValue::Term(Term::Value(Value::Integer(42))),
827                location: 1..3,
828            }))
829        );
830        assert_eq!(
831            Tokens::new("042 ").next(),
832            Some(Ok(Token {
833                value: TokenValue::Term(Term::Value(Value::Integer(0o42))),
834                location: 0..3,
835            }))
836        );
837        assert_eq!(
838            Tokens::new("\t 123 \n").next(),
839            Some(Ok(Token {
840                value: TokenValue::Term(Term::Value(Value::Integer(123))),
841                location: 2..5,
842            }))
843        );
844    }
845
846    #[test]
847    fn parsing_two_tokens() {
848        let mut tokens = Tokens::new(" 123  foo ");
849        assert_eq!(
850            tokens.next(),
851            Some(Ok(Token {
852                value: TokenValue::Term(Term::Value(Value::Integer(123))),
853                location: 1..4,
854            }))
855        );
856        assert_eq!(
857            tokens.next(),
858            Some(Ok(Token {
859                value: TokenValue::Term(Term::Variable {
860                    name: "foo",
861                    location: 6..9,
862                }),
863                location: 6..9,
864            }))
865        );
866        assert_eq!(
867            tokens.next(),
868            Some(Ok(Token {
869                value: TokenValue::EndOfInput,
870                location: 10..10,
871            }))
872        );
873    }
874
875    #[test]
876    fn parsing_many_tokens() {
877        // TODO "10.0e+3+0"
878        let mut tokens = Tokens::new(" 10+0 ");
879        assert_eq!(
880            tokens.next(),
881            Some(Ok(Token {
882                value: TokenValue::Term(Term::Value(Value::Integer(10))),
883                location: 1..3,
884            }))
885        );
886        assert_eq!(
887            tokens.next(),
888            Some(Ok(Token {
889                value: TokenValue::Operator(Operator::Plus),
890                location: 3..4,
891            }))
892        );
893        assert_eq!(
894            tokens.next(),
895            Some(Ok(Token {
896                value: TokenValue::Term(Term::Value(Value::Integer(0))),
897                location: 4..5,
898            }))
899        );
900        assert_eq!(
901            tokens.next(),
902            Some(Ok(Token {
903                value: TokenValue::EndOfInput,
904                location: 6..6,
905            }))
906        );
907    }
908
909    #[test]
910    fn parsing_adjacent_operators() {
911        let mut tokens = Tokens::new("+-0");
912        assert_eq!(
913            tokens.next(),
914            Some(Ok(Token {
915                value: TokenValue::Operator(Operator::Plus),
916                location: 0..1,
917            }))
918        );
919        assert_eq!(
920            tokens.next(),
921            Some(Ok(Token {
922                value: TokenValue::Operator(Operator::Minus),
923                location: 1..2,
924            }))
925        );
926        assert_eq!(
927            tokens.next(),
928            Some(Ok(Token {
929                value: TokenValue::Term(Term::Value(Value::Integer(0))),
930                location: 2..3,
931            }))
932        );
933        assert_eq!(
934            tokens.next(),
935            Some(Ok(Token {
936                value: TokenValue::EndOfInput,
937                location: 3..3,
938            }))
939        );
940    }
941
942    #[test]
943    fn unrecognized_character() {
944        assert_eq!(
945            Tokens::new("#").next(),
946            Some(Err(Error {
947                cause: TokenError::InvalidCharacter,
948                location: 0..1,
949            }))
950        );
951        assert_eq!(
952            Tokens::new(" @@").next(),
953            Some(Err(Error {
954                cause: TokenError::InvalidCharacter,
955                location: 1..2,
956            }))
957        );
958    }
959
960    #[test]
961    fn peekable_tokens() {
962        let mut tokens = PeekableTokens::from("1 + 2");
963        assert_eq!(
964            tokens.peek(),
965            &Ok(Token {
966                value: TokenValue::Term(Term::Value(Value::Integer(1))),
967                location: 0..1,
968            })
969        );
970        assert_eq!(
971            tokens.peek(),
972            &Ok(Token {
973                value: TokenValue::Term(Term::Value(Value::Integer(1))),
974                location: 0..1,
975            })
976        );
977        assert_eq!(
978            tokens.next(),
979            Ok(Token {
980                value: TokenValue::Term(Term::Value(Value::Integer(1))),
981                location: 0..1,
982            })
983        );
984
985        assert_eq!(
986            tokens.peek(),
987            &Ok(Token {
988                value: TokenValue::Operator(Operator::Plus),
989                location: 2..3,
990            })
991        );
992        assert_eq!(
993            tokens.next(),
994            Ok(Token {
995                value: TokenValue::Operator(Operator::Plus),
996                location: 2..3,
997            })
998        );
999
1000        assert_eq!(
1001            tokens.next(),
1002            Ok(Token {
1003                value: TokenValue::Term(Term::Value(Value::Integer(2))),
1004                location: 4..5,
1005            })
1006        );
1007
1008        assert_eq!(
1009            tokens.peek(),
1010            &Ok(Token {
1011                value: TokenValue::EndOfInput,
1012                location: 5..5,
1013            })
1014        );
1015        assert_eq!(
1016            tokens.next(),
1017            Ok(Token {
1018                value: TokenValue::EndOfInput,
1019                location: 5..5,
1020            })
1021        );
1022    }
1023}
yash_arith/token.rs

yash_arith/
token.rs
