dbt_antlr_runtime/input_stream.rs
1//! Input to lexer
2use std::borrow::Cow;
3
4use crate::char_stream::{CharStream, InputData};
5use crate::int_stream::IntStream;
6
7/// Default rust target input stream.
8///
9/// Since Rust uses UTF-8 format which does not support indexing by char,
10/// `InputStream<&str>` has slightly different index behavior in compare to java runtime when there are
11/// non-ASCII unicode characters.
12/// If you need it to generate exactly the same indexes as Java runtime, you have to use `CodePoint8/16/32BitCharStream`,
13/// which does not use rusts native `str` type, so it would do additional conversions and allocations along the way.
14#[derive(Debug)]
15pub struct InputStream<'input> {
16 name: String,
17 data_raw: &'input str,
18 index: isize,
19}
20
21impl<'input> CharStream<'input> for InputStream<'input> {
22 #[inline]
23 fn get_text(&self, start: isize, stop: isize) -> Cow<'input, str> {
24 self.get_text_inner(start, stop).into()
25 }
26}
27
28// /// `InputStream` over byte slice
29// pub type ByteStream<'a> = InputStream<&'a [u8]>;
30// /// InputStream which treats the input as a series of Unicode code points that fit into `u8`
31// pub type CodePoint8BitCharStream<'a> = InputStream<&'a [u8]>;
32// /// InputStream which treats the input as a series of Unicode code points that fit into `u16`
33// pub type CodePoint16BitCharStream<'a> = InputStream<&'a [u16]>;
34// /// InputStream which treats the input as a series of Unicode code points
35// pub type CodePoint32BitCharStream<'a> = InputStream<&'a [u32]>;
36
37// impl<'a, T> CharStream<Cow<'a, [T]>> for InputStream<&'a [T]>
38// where
39// [T]: InputData,
40// {
41// #[inline]
42// fn get_text(&self, a: isize, b: isize) -> Cow<'a, [T]> {
43// Cow::Borrowed(self.get_text_inner(a, b))
44// }
45// }
46
47// impl<T> CharStream<String> for InputStream<&[T]>
48// where
49// [T]: InputData,
50// {
51// fn get_text(&self, a: isize, b: isize) -> String {
52// self.get_text_inner(a, b).to_display()
53// }
54// }
55
56// impl<'b, T> CharStream<Cow<'b, str>> for InputStream<&[T]>
57// where
58// [T]: InputData,
59// {
60// #[inline]
61// fn get_text(&self, a: isize, b: isize) -> Cow<'b, str> {
62// self.get_text_inner(a, b).to_display().into()
63// }
64// }
65
66// impl<'a, T> CharStream<&'a [T]> for InputStream<&'a [T]>
67// where
68// [T]: InputData,
69// {
70// #[inline]
71// fn get_text(&self, a: isize, b: isize) -> &'a [T] {
72// self.get_text_inner(a, b)
73// }
74// }
75
76// impl<Data: ?Sized + InputData> InputStream<Box<Data>> {
77// fn get_text_owned(&self, start: isize, stop: isize) -> Data::Owned {
78// let start = start as usize;
79// let stop = self.data_raw.offset(stop, 1).unwrap_or(stop) as usize;
80
81// if stop < self.data_raw.len() {
82// &self.data_raw[start..stop]
83// } else {
84// &self.data_raw[start..]
85// }
86// .to_owned()
87// }
88
89// /// Creates new `InputStream` over owned data
90// pub fn new_owned(data: Box<Data>) -> Self {
91// Self {
92// name: "<empty>".to_string(),
93// data_raw: data,
94// index: 0,
95// }
96// }
97// }
98
99impl<'input> InputStream<'input> {
100 fn get_text_inner(&self, start: isize, stop: isize) -> &'input str {
101 // println!("get text {}..{} of {:?}",start,stop,self.data_raw.to_display());
102 let start = start as usize;
103 let stop = self.data_raw.offset(stop, 1).unwrap_or(stop) as usize;
104 // println!("justed range {}..{} ",start,stop);
105 // let start = self.data_raw.offset(0,start).unwrap() as usize;
106 // let stop = self.data_raw.offset(0,stop + 1).unwrap() as usize;
107
108 if stop < self.data_raw.len() {
109 &self.data_raw[start..stop]
110 } else {
111 &self.data_raw[start..]
112 }
113 }
114
115 /// Creates new `InputStream` over borrowed data
116 pub fn new(data_raw: &'input str) -> Self {
117 // let data_raw = data_raw.as_ref();
118 // let data = data_raw.to_indexed_vec();
119 Self {
120 name: "<empty>".to_owned(),
121 data_raw,
122 index: 0,
123 // phantom: Default::default(),
124 }
125 }
126
127 /// Resets input stream to start from the beginning of this slice
128 #[inline]
129 pub fn reset(&mut self) {
130 self.index = 0
131 }
132}
133
134impl IntStream for InputStream<'_> {
135 #[inline]
136 fn consume(&mut self) {
137 if let Some(index) = self.data_raw.offset(self.index, 1) {
138 self.index = index;
139 // self.current = self.data_raw.deref().item(index).unwrap_or(TOKEN_EOF);
140 // Ok(())
141 } else {
142 panic!("cannot consume EOF");
143 }
144 }
145
146 #[inline]
147 fn la(&mut self, mut offset: isize) -> i32 {
148 if offset == 1 {
149 return self
150 .data_raw
151 .item(self.index)
152 .unwrap_or(crate::int_stream::EOF);
153 }
154 assert!(offset != 0, "should not be called with offset 0");
155 if offset < 0 {
156 offset += 1; // e.g., translate LA(-1) to use offset i=0; then data[p+0-1]
157 }
158
159 self.data_raw
160 .offset(self.index, offset - 1)
161 .and_then(|index| self.data_raw.item(index))
162 .unwrap_or(crate::int_stream::EOF)
163 }
164
165 #[inline]
166 fn mark(&mut self) -> isize {
167 -1
168 }
169
170 #[inline]
171 fn release(&mut self, _marker: isize) {}
172
173 #[inline]
174 fn index(&self) -> isize {
175 self.index
176 }
177
178 #[inline]
179 fn seek(&mut self, index: isize) {
180 self.index = index
181 }
182
183 #[inline]
184 fn size(&self) -> isize {
185 self.data_raw.len() as isize
186 }
187
188 fn get_source_name(&self) -> String {
189 self.name.clone()
190 }
191}
192
193#[cfg(test)]
194mod test {
195 use crate::{char_stream::CharStream, int_stream::EOF};
196
197 use super::InputStream;
198
199 #[test]
200 fn test_str_input_stream() {
201 let mut input = InputStream::new("V1は3");
202 let input = &mut input as &mut dyn CharStream;
203 assert_eq!(input.la(1), 'V' as i32);
204 assert_eq!(input.index(), 0);
205 input.consume();
206 assert_eq!(input.la(1), '1' as i32);
207 assert_eq!(input.la(-1), 'V' as i32);
208 assert_eq!(input.index(), 1);
209 input.consume();
210 assert_eq!(input.la(1), 0x306F);
211 assert_eq!(input.index(), 2);
212 input.consume();
213 assert_eq!(input.index(), 5);
214 assert_eq!(input.la(-2), '1' as i32);
215 assert_eq!(input.la(2), EOF);
216 assert_eq!(input.get_text(1, 1), "1");
217 assert_eq!(input.get_text(1, 2), "1は");
218 assert_eq!(input.get_text(2, 2), "は");
219 assert_eq!(input.get_text(2, 5), "は3");
220 assert_eq!(input.get_text(5, 5), "3");
221 }
222
223 // #[test]
224 // fn test_byte_input_stream() {
225 // let mut input = InputStream::new(&b"V\xaa\xbb"[..]);
226 // assert_eq!(input.la(1), 'V' as i32);
227 // input.seek(2);
228 // assert_eq!(input.la(1), 0xBB);
229 // assert_eq!(input.index(), 2);
230 // let mut input = InputStream::new("は".as_bytes());
231 // assert_eq!(input.la(1), 227);
232 // }
233}