docbuf_core/
lexer.rs

1use std::collections::HashMap;
2use std::ops::Range;
3
4pub use logos::{Lexer, Logos};
5
6// Keep track of the latest token ranges.
7pub type TokenRanges = HashMap<Token, Range<usize>>;
8
9#[derive(Logos, Debug, PartialEq, Clone)]
10pub enum Token {
11    // Search for the pragma version
12    #[token("pragma docbuf")]
13    Pragma,
14    // Search for the module name
15    #[token("\nmodule")]
16    Module,
17    // Search for imports
18    #[token("\nimport")]
19    Import,
20    // Search for regex matching `document <name> {`
21    #[token("\ndocument")]
22    Document,
23    // Search for enumerable items
24    #[token("\nenumerable")]
25    Enumerate,
26    // Search for process statements
27    #[token("\nprocess")]
28    Process,
29    // // Search for document, field, process names
30    // #[regex("[a-zA-Z0-9]+")]
31    // Name,
32    // Search for option assignments
33    #[token(" = ")]
34    StatementAssign,
35    // Search for the end of a statement
36    #[token(";")]
37    StatementEnd,
38    // Search for document options
39    #[token("#[document::options {")]
40    DocumentOptionsStart,
41    // Search for field options
42    #[token("#[field::options {")]
43    FieldOptionsStart,
44    // Search for enum options
45    #[token("#[enum::options {")]
46    EnumOptionsStart,
47    // Search for item options
48    #[token("#[item::options {")]
49    ItemOptionsStart,
50    // Search for process options
51    #[token("#[process::options {")]
52    ProcessOptionsStart,
53    // Search for the end of options
54    #[token("\n}]")]
55    OptionsEnd,
56    // Search for the start of a section
57    #[token("{")]
58    SectionStart,
59    // Search for the end of a section
60    #[token("\n}")]
61    SectionEnd,
62    // Search for the field delimiter
63    #[token(":")]
64    FieldDelimiter,
65    // Search for the end of the field
66    #[token(",")]
67    FieldEnd,
68    // Search for a comment line
69    #[token("// ")]
70    CommentLine,
71    // Search for a documentation comment line
72    #[token("/// ")]
73    DocCommentLine,
74    // Search for a new line
75    #[regex(r"\n", |lex| lex.bump(1))]
76    NewLine,
77}
docbuf_core/lexer.rs

docbuf_core/
lexer.rs
