do_not_use_antlr_rust/
char_stream.rs

1//! `IntStream` extension for Lexer that allows subslicing of underlying data
2use std::char::REPLACEMENT_CHARACTER;
3use std::convert::TryFrom;
4use std::fmt::Debug;
5use std::ops::{Index, Range, RangeFrom};
6
7use crate::int_stream::IntStream;
8
9/// Provides underlying data for Tokens.
10pub trait CharStream<Data>: IntStream {
11    /// Returns underlying data piece, either slice or owned copy.
12    /// Panics if provided indexes are invalid
13    /// Called by parser only on token intervals.
14    /// This fact can be used by custom implementations  
15    fn get_text(&self, a: isize, b: isize) -> Data;
16}
17
18/// Trait for input that can be accepted by `InputStream` to be able to provide lexer with data.
19/// Public for implementation reasons.
20pub trait InputData:
21    Index<Range<usize>, Output = Self>
22    + Index<RangeFrom<usize>, Output = Self>
23    + ToOwned
24    + Debug
25    + 'static
26{
27    // fn to_indexed_vec(&self) -> Vec<(u32, u32)>;
28
29    #[doc(hidden)]
30    fn offset(&self, index: isize, item_offset: isize) -> Option<isize>;
31
32    #[doc(hidden)]
33    fn item(&self, index: isize) -> Option<isize>;
34
35    #[doc(hidden)]
36    fn len(&self) -> usize;
37
38    #[doc(hidden)]
39    fn from_text(text: &str) -> Self::Owned;
40
41    #[doc(hidden)]
42    fn to_display(&self) -> String;
43}
44
45impl<T: Into<u32> + From<u8> + TryFrom<u32> + Copy + Debug + 'static> InputData for [T]
46where
47    <T as TryFrom<u32>>::Error: Debug,
48{
49    // fn to_indexed_vec(&self) -> Vec<(u32, u32)> {
50    //     self.into_iter()
51    //         .enumerate()
52    //         .map(|(x, &y)| (x as u32, y.into()))
53    //         .collect()
54    // }
55
56    #[inline]
57    fn offset(&self, index: isize, item_offset: isize) -> Option<isize> {
58        let new_index = index + item_offset;
59        if new_index < 0 {
60            return None; // invalid; no char before first char
61        }
62        if new_index > self.len() as isize {
63            return None;
64        }
65
66        Some(new_index)
67    }
68
69    #[inline]
70    fn item(&self, index: isize) -> Option<isize> {
71        self.get(index as usize).map(|&it| it.into() as isize)
72    }
73
74    #[inline]
75    fn len(&self) -> usize { self.len() }
76
77    #[inline]
78    fn from_text(text: &str) -> Self::Owned {
79        text.chars()
80            .map(|it| T::try_from(it as u32).unwrap())
81            .collect()
82    }
83
84    #[inline]
85    // default
86    fn to_display(&self) -> String {
87        self.iter()
88            .map(|x| char::try_from((*x).into()).unwrap_or(REPLACEMENT_CHARACTER))
89            .collect()
90    }
91}
92//
93// impl InputData for [u8] {
94//     #[inline]
95//     fn to_display(&self) -> String { String::from_utf8_lossy(self).into_owned() }
96// }
97
98// impl InputData for [u16] {
99// }
100//
101// impl InputData for [u32] {
102//     #[inline]
103//     fn to_display(&self) -> String {
104//         self.iter()
105//             .map(|x| char::try_from(*x).unwrap_or(REPLACEMENT_CHARACTER))
106//             .collect()
107//     }
108// }
109
110impl InputData for str {
111    // fn to_indexed_vec(&self) -> Vec<(u32, u32)> {
112    //     self.char_indices()
113    //         .map(|(i, ch)| (i as u32, ch as u32))
114    //         .collect()
115    // }
116
117    #[inline]
118    fn offset(&self, mut index: isize, mut item_offset: isize) -> Option<isize> {
119        if item_offset == 0 {
120            return Some(index);
121        }
122        let direction = item_offset.signum();
123
124        while {
125            index += direction;
126            if index < 0 || index > self.len() as isize {
127                return None;
128            }
129            if self.is_char_boundary(index as usize) {
130                item_offset -= direction;
131            }
132            item_offset != 0
133        } {}
134
135        Some(index)
136    }
137
138    #[inline]
139    fn item(&self, index: isize) -> Option<isize> {
140        self.get(index as usize..)
141            .and_then(|it| it.chars().next())
142            .map(|it| it as isize)
143    }
144
145    #[inline]
146    fn len(&self) -> usize { self.len() }
147
148    fn from_text(text: &str) -> Self::Owned { text.to_owned() }
149
150    // #[inline]
151    // fn from_text(text: &str) -> Self::Owned { text.to_owned() }
152
153    #[inline]
154    fn to_display(&self) -> String { self.to_string() }
155}
do_not_use_antlr_rust/char_stream.rs

do_not_use_antlr_rust/
char_stream.rs
