lexer_rs/lexer/
lexer_of_string.rs

1//a Imports
2use std::marker::PhantomData;
3
4use crate::{FmtContext, PosnInCharStream};
5use crate::{LexerError, LexerOfStr};
6
7//a LexerOfString
8//tp LexerOfString
9/// This provides a type that wraps an allocated [String], and which
10/// tracks the lines within the string. It then provides a method to
11/// create a [LexerOfStr] that borrows the text, and which can the be
12/// used as a [crate::Lexer].
13///
14/// This type also implements the [FmtContext] trait, which allows for
15/// pretty-printing the text between certain lines, to highlight certain
16/// characters or regions of the text.
17///
18/// Should the [LexerOfStr] return an error while parsing then the
19/// [FmtContext] implementation can be used to generate a very useful
20/// error message with the context of the error.
21///
22/// This also applies should the [LexerOfStr] output be used in a further
23/// grammar - the methods to pretty-print the text require just start
24/// and end positions of the position type, which must support
25/// [PosnInCharStream].
26///
27/// The [LexerOfString] may be used more than once, replacing its text
28/// if necessary before a new parse is initiated. The [String] can
29/// also be retrieved, which is useful for droppign the
30/// [LexerOfString] but retaining the text for future parse or
31/// compilation stages
32///
33#[derive(Debug)]
34pub struct LexerOfString<P, T, E>
35where
36    P: PosnInCharStream,
37{
38    text: String,
39    line_start_ncolumns: Vec<(P, usize)>,
40    _phantom_token: PhantomData<T>,
41    _phantom_error: PhantomData<E>,
42}
43
44//ip LexerOfString
45impl<P, T, E> Default for LexerOfString<P, T, E>
46where
47    P: PosnInCharStream,
48{
49    fn default() -> Self {
50        Self {
51            text: String::new(),
52            line_start_ncolumns: Vec::new(),
53            _phantom_token: PhantomData,
54            _phantom_error: PhantomData,
55        }
56    }
57}
58
59//ip LexerOfString
60impl<P, T, E> LexerOfString<P, T, E>
61where
62    P: PosnInCharStream,
63    T: std::fmt::Debug + Clone,
64    E: LexerError<P>,
65{
66    //cp set_text
67    /// Set the text
68    pub fn set_text<S: Into<String>>(mut self, text: S) -> Self {
69        self.text = text.into();
70        self.find_line_starts();
71        self
72    }
73
74    //mp take_text
75    /// Take the text as a [String] out of the [LexerOfString]
76    pub fn take_text(&mut self) -> String {
77        self.line_start_ncolumns.clear();
78        std::mem::take(&mut self.text)
79    }
80
81    //mp text
82    /// Get a [str] reference to the text content
83    pub fn text(&self) -> &str {
84        &self.text
85    }
86
87    //mp lexer
88    /// Create a [LexerOfStr] that will parse the text
89    pub fn lexer(&self) -> LexerOfStr<P, T, E> {
90        LexerOfStr::new(&self.text)
91    }
92
93    //mi find_line_starts
94    /// Finds the start byte offset and number of columns for all the
95    /// lines in the text
96    fn find_line_starts(&mut self) {
97        let mut line_start_ncolumns = Vec::new();
98        let mut s: &str = &self.text;
99        let mut pos = P::default();
100        line_start_ncolumns.push((pos, 0)); // Line '0'
101        while let Some((line, next_line)) = s.split_once('\n') {
102            let ncolumns = line.chars().count();
103            line_start_ncolumns.push((pos, ncolumns));
104            pos = pos.advance_line(line.len() + 1);
105            s = next_line;
106        }
107        let ncolumns = s.chars().count();
108        line_start_ncolumns.push((pos, ncolumns));
109        self.line_start_ncolumns = line_start_ncolumns;
110    }
111}
112
113//a Impl FmtContext
114//ip FmtContext for LexerOfString
115impl<P, T, E> FmtContext<P> for LexerOfString<P, T, E>
116where
117    P: PosnInCharStream,
118    E: LexerError<P>,
119{
120    fn line_length(&self, line: usize) -> usize {
121        self.line_start_ncolumns[line].1
122    }
123
124    fn fmt_line(&self, f: &mut dyn std::fmt::Write, line: usize) -> std::fmt::Result {
125        let s = &self.text[self.line_start_ncolumns[line].0.byte_ofs()..];
126        let s = s.split_once('\n').map(|(s, _)| s).unwrap_or(s);
127        write!(f, "{}", s)
128    }
129}
lexer_rs/lexer/lexer_of_string.rs

lexer_rs/lexer/
lexer_of_string.rs
