1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
---
source: crates/compiler/src/lexer.rs
expression: "tokenize(\"0 23 1.1\")"
---
(
[
Token {
kind: Integer,
text: "0",
range: 0..1,
},
Token {
kind: Whitespace,
text: " ",
range: 1..2,
},
Token {
kind: Integer,
text: "23",
range: 2..4,
},
Token {
kind: Whitespace,
text: " ",
range: 4..5,
},
Token {
kind: Float,
text: "1.1",
range: 5..8,
},
Token {
kind: Eof,
text: "",
range: 8..8,
},
],
[],
)