scanner_rust/scanner.rs
1use std::{
2 char::REPLACEMENT_CHARACTER,
3 fs::File,
4 io::{ErrorKind, Read},
5 path::Path,
6 ptr::copy,
7 str::{FromStr, from_utf8, from_utf8_unchecked},
8};
9
10use educe::Educe;
11use utf8_width::*;
12
13use crate::{ScannerError, kmp::compute_lps, whitespaces::*};
14
15/// A simple text scanner which can parse primitive types and strings using UTF-8.
16#[derive(Educe)]
17#[educe(Debug)]
18pub struct Scanner<R: Read, const N: usize = 256> {
19 #[educe(Debug(ignore))]
20 reader: R,
21 buf: [u8; N],
22 buf_length: usize,
23 buf_offset: usize,
24 passing_byte: Option<u8>,
25}
26
27impl<R: Read> Scanner<R> {
28 /// Create a scanner from a reader.
29 ///
30 /// ```rust
31 /// use std::io;
32 ///
33 /// use scanner_rust::Scanner;
34 ///
35 /// let mut sc = Scanner::new(io::stdin());
36 /// ```
37 #[inline]
38 pub fn new(reader: R) -> Scanner<R> {
39 Self::new2(reader)
40 }
41}
42
43impl<R: Read, const N: usize> Scanner<R, N> {
44 /// Create a scanner from a reader and set the buffer size via generics.
45 ///
46 /// ```rust
47 /// use std::io;
48 ///
49 /// use scanner_rust::Scanner;
50 ///
51 /// let mut sc: Scanner<_, 1024> = Scanner::new2(io::stdin());
52 /// ```
53 #[inline]
54 pub fn new2(reader: R) -> Scanner<R, N> {
55 // The buffer must be at least 4 bytes to hold a full UTF-8 character.
56 const { assert!(N >= 4, "the buffer size N must be at least 4 bytes") };
57
58 Scanner {
59 reader,
60 buf: [0u8; N],
61 buf_length: 0,
62 buf_offset: 0,
63 passing_byte: None,
64 }
65 }
66}
67
68impl Scanner<File> {
69 /// Create a scanner to read data from a file by its path.
70 ///
71 /// ```rust
72 /// use scanner_rust::Scanner;
73 ///
74 /// let mut sc = Scanner::scan_path("Cargo.toml").unwrap();
75 /// ```
76 #[inline]
77 pub fn scan_path<P: AsRef<Path>>(path: P) -> Result<Scanner<File>, ScannerError> {
78 Self::scan_path2(path)
79 }
80}
81
82impl<const N: usize> Scanner<File, N> {
83 /// Create a scanner to read data from a file by its path and set the buffer size via generics.
84 ///
85 /// ```rust
86 /// use scanner_rust::Scanner;
87 ///
88 /// let mut sc: Scanner<_, 1024> = Scanner::scan_path2("Cargo.toml").unwrap();
89 /// ```
90 #[inline]
91 pub fn scan_path2<P: AsRef<Path>>(path: P) -> Result<Scanner<File, N>, ScannerError> {
92 let reader = File::open(path)?;
93
94 Ok(Scanner::new2(reader))
95 }
96}
97
98impl<R: Read, const N: usize> Scanner<R, N> {
99 #[inline]
100 fn buf_align_to_front_end(&mut self) {
101 unsafe {
102 copy(self.buf.as_ptr().add(self.buf_offset), self.buf.as_mut_ptr(), self.buf_length);
103 }
104
105 self.buf_offset = 0;
106 }
107
108 #[inline]
109 fn buf_left_shift(&mut self, distance: usize) {
110 debug_assert!(self.buf_length >= distance);
111
112 self.buf_offset += distance;
113
114 if self.buf_offset >= N - 4 {
115 self.buf_align_to_front_end();
116 }
117
118 self.buf_length -= distance;
119 }
120
121 /// Left shift (if necessary) the buffer to remove bytes from the start of the buffer. Typically, you should use this after `peek`ing the buffer.
122 ///
123 /// # Safety
124 ///
125 /// `number_of_bytes` must not be greater than the length of the currently buffered data (the length of the slice returned by `peek`). A larger value underflows the internal buffer length and causes out-of-bounds access afterwards.
126 #[inline]
127 pub unsafe fn remove_heading_bytes_from_buffer(&mut self, number_of_bytes: usize) {
128 self.buf_left_shift(number_of_bytes);
129 }
130
131 fn passing_read(&mut self) -> Result<bool, ScannerError> {
132 if self.buf_length == 0 {
133 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
134
135 if size == 0 {
136 return Ok(false);
137 }
138
139 self.buf_length += size;
140
141 if let Some(passing_byte) = self.passing_byte.take()
142 && self.buf[self.buf_offset] == passing_byte
143 {
144 self.buf_left_shift(1);
145
146 return if size == 1 {
147 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
148
149 if size == 0 {
150 Ok(false)
151 } else {
152 self.buf_length += size;
153
154 Ok(true)
155 }
156 } else {
157 Ok(true)
158 };
159 }
160
161 Ok(true)
162 } else {
163 Ok(true)
164 }
165 }
166}
167
168impl<R: Read, const N: usize> Scanner<R, N> {
169 /// Read the next char. If the data is not a correct char, it will return a `Ok(Some(REPLACEMENT_CHARACTER))` which is �. If there is nothing to read, it will return `Ok(None)`.
170 ///
171 /// ```rust
172 /// use scanner_rust::Scanner;
173 ///
174 /// let mut sc = Scanner::new("5 c 中文".as_bytes());
175 ///
176 /// assert_eq!(Some('5'), sc.next_char().unwrap());
177 /// assert_eq!(Some(' '), sc.next_char().unwrap());
178 /// assert_eq!(Some('c'), sc.next_char().unwrap());
179 /// assert_eq!(Some(' '), sc.next_char().unwrap());
180 /// assert_eq!(Some('中'), sc.next_char().unwrap());
181 /// assert_eq!(Some('文'), sc.next_char().unwrap());
182 /// assert_eq!(None, sc.next_char().unwrap());
183 /// ```
184 pub fn next_char(&mut self) -> Result<Option<char>, ScannerError> {
185 if !self.passing_read()? {
186 return Ok(None);
187 }
188
189 let e = self.buf[self.buf_offset];
190
191 let width = get_width(e);
192
193 match width {
194 0 => {
195 self.buf_left_shift(1);
196
197 Ok(Some(REPLACEMENT_CHARACTER))
198 },
199 1 => {
200 self.buf_left_shift(1);
201
202 Ok(Some(e as char))
203 },
204 _ => {
205 while self.buf_length < width {
206 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..]) {
207 Ok(0) => {
208 self.buf_left_shift(1);
209
210 return Ok(Some(REPLACEMENT_CHARACTER));
211 },
212 Ok(c) => self.buf_length += c,
213 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
214 Err(err) => return Err(err.into()),
215 }
216 }
217
218 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
219
220 match from_utf8(char_str_bytes) {
221 Ok(char_str) => {
222 let c = char_str.chars().next();
223
224 self.buf_left_shift(width);
225
226 Ok(c)
227 },
228 Err(_) => {
229 self.buf_left_shift(1);
230
231 Ok(Some(REPLACEMENT_CHARACTER))
232 },
233 }
234 },
235 }
236 }
237
238 /// Read the next line but not include the trailing line character (or line characters like `CrLf`(`\r\n`)). If there is nothing to read, it will return `Ok(None)`.
239 ///
240 /// ```rust
241 /// use scanner_rust::Scanner;
242 ///
243 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
244 ///
245 /// assert_eq!(Some("123 456".into()), sc.next_line().unwrap());
246 /// assert_eq!(Some("789 ".into()), sc.next_line().unwrap());
247 /// assert_eq!(Some("".into()), sc.next_line().unwrap());
248 /// assert_eq!(Some(" 中文 ".into()), sc.next_line().unwrap());
249 /// ```
250 pub fn next_line(&mut self) -> Result<Option<String>, ScannerError> {
251 if !self.passing_read()? {
252 return Ok(None);
253 }
254
255 let mut temp = String::new();
256
257 loop {
258 let e = self.buf[self.buf_offset];
259
260 let width = get_width(e);
261
262 match width {
263 0 => {
264 self.buf_left_shift(1);
265
266 temp.push(REPLACEMENT_CHARACTER);
267 },
268 1 => {
269 match e {
270 b'\n' => {
271 if self.buf_length == 1 {
272 self.passing_byte = Some(b'\r');
273 self.buf_left_shift(1);
274 } else if self.buf[self.buf_offset + 1] == b'\r' {
275 self.buf_left_shift(2);
276 } else {
277 self.buf_left_shift(1);
278 }
279
280 return Ok(Some(temp));
281 },
282 b'\r' => {
283 if self.buf_length == 1 {
284 self.passing_byte = Some(b'\n');
285 self.buf_left_shift(1);
286 } else if self.buf[self.buf_offset + 1] == b'\n' {
287 self.buf_left_shift(2);
288 } else {
289 self.buf_left_shift(1);
290 }
291
292 return Ok(Some(temp));
293 },
294 _ => (),
295 }
296
297 self.buf_left_shift(1);
298
299 temp.push(e as char);
300 },
301 _ => {
302 while self.buf_length < width {
303 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
304 {
305 Ok(0) => {
306 temp.push_str(
307 String::from_utf8_lossy(
308 &self.buf
309 [self.buf_offset..(self.buf_offset + self.buf_length)],
310 )
311 .as_ref(),
312 );
313
314 self.buf_left_shift(self.buf_length);
315
316 return Ok(Some(temp));
317 },
318 Ok(c) => self.buf_length += c,
319 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
320 Err(err) => return Err(err.into()),
321 }
322 }
323
324 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
325
326 match from_utf8(char_str_bytes) {
327 Ok(char_str) => {
328 temp.push_str(char_str);
329
330 self.buf_left_shift(width);
331 },
332 Err(_) => {
333 self.buf_left_shift(1);
334
335 temp.push(REPLACEMENT_CHARACTER);
336 },
337 }
338 },
339 }
340
341 if self.buf_length == 0 {
342 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
343
344 if size == 0 {
345 return Ok(Some(temp));
346 }
347
348 self.buf_length += size;
349 }
350 }
351 }
352
353 /// Read the next line but not include the trailing line character (or line characters like `CrLf`(`\r\n`)) without fully validating UTF-8. If there is nothing to read, it will return `Ok(None)`.
354 ///
355 /// ```rust
356 /// use scanner_rust::Scanner;
357 ///
358 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
359 ///
360 /// assert_eq!(Some("123 456".into()), sc.next_line_raw().unwrap());
361 /// assert_eq!(Some("789 ".into()), sc.next_line_raw().unwrap());
362 /// assert_eq!(Some("".into()), sc.next_line_raw().unwrap());
363 /// assert_eq!(Some(" 中文 ".into()), sc.next_line_raw().unwrap());
364 /// ```
365 pub fn next_line_raw(&mut self) -> Result<Option<Vec<u8>>, ScannerError> {
366 if !self.passing_read()? {
367 return Ok(None);
368 }
369
370 let mut temp = Vec::new();
371
372 loop {
373 let e = self.buf[self.buf_offset];
374
375 let width = get_width(e);
376
377 match width {
378 0 => {
379 self.buf_left_shift(1);
380
381 temp.push(e);
382 },
383 1 => {
384 match e {
385 b'\n' => {
386 if self.buf_length == 1 {
387 self.passing_byte = Some(b'\r');
388 self.buf_left_shift(1);
389 } else if self.buf[self.buf_offset + 1] == b'\r' {
390 self.buf_left_shift(2);
391 } else {
392 self.buf_left_shift(1);
393 }
394
395 return Ok(Some(temp));
396 },
397 b'\r' => {
398 if self.buf_length == 1 {
399 self.passing_byte = Some(b'\n');
400 self.buf_left_shift(1);
401 } else if self.buf[self.buf_offset + 1] == b'\n' {
402 self.buf_left_shift(2);
403 } else {
404 self.buf_left_shift(1);
405 }
406
407 return Ok(Some(temp));
408 },
409 _ => (),
410 }
411
412 self.buf_left_shift(1);
413
414 temp.push(e);
415 },
416 _ => {
417 while self.buf_length < width {
418 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
419 {
420 Ok(0) => {
421 temp.extend_from_slice(
422 &self.buf[self.buf_offset..(self.buf_offset + self.buf_length)],
423 );
424
425 self.buf_left_shift(self.buf_length);
426
427 return Ok(Some(temp));
428 },
429 Ok(c) => self.buf_length += c,
430 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
431 Err(err) => return Err(err.into()),
432 }
433 }
434
435 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
436
437 temp.extend_from_slice(char_str_bytes);
438
439 self.buf_left_shift(width);
440 },
441 }
442
443 if self.buf_length == 0 {
444 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
445
446 if size == 0 {
447 return Ok(Some(temp));
448 }
449
450 self.buf_length += size;
451 }
452 }
453 }
454
455 /// Drop the next line but not include the trailing line character (or line characters like `CrLf`(`\r\n`)). If there is nothing to read, it will return `Ok(None)`. If there is something to read, it will return `Ok(Some(i))`. The `i` is the length of the dropped line.
456 ///
457 /// ```rust
458 /// use scanner_rust::Scanner;
459 ///
460 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
461 ///
462 /// assert_eq!(Some(7), sc.drop_next_line().unwrap());
463 /// assert_eq!(Some("789 ".into()), sc.next_line().unwrap());
464 /// assert_eq!(Some(0), sc.drop_next_line().unwrap());
465 /// assert_eq!(Some(" 中文 ".into()), sc.next_line().unwrap());
466 /// assert_eq!(None, sc.drop_next_line().unwrap());
467 /// ```
468 pub fn drop_next_line(&mut self) -> Result<Option<usize>, ScannerError> {
469 if !self.passing_read()? {
470 return Ok(None);
471 }
472
473 let mut c = 0;
474
475 loop {
476 let e = self.buf[self.buf_offset];
477
478 let width = get_width(e);
479
480 match width {
481 0 => {
482 self.buf_left_shift(1);
483
484 c += 1;
485 },
486 1 => {
487 match e {
488 b'\n' => {
489 if self.buf_length == 1 {
490 self.passing_byte = Some(b'\r');
491 self.buf_left_shift(1);
492 } else if self.buf[self.buf_offset + 1] == b'\r' {
493 self.buf_left_shift(2);
494 } else {
495 self.buf_left_shift(1);
496 }
497
498 return Ok(Some(c));
499 },
500 b'\r' => {
501 if self.buf_length == 1 {
502 self.passing_byte = Some(b'\n');
503 self.buf_left_shift(1);
504 } else if self.buf[self.buf_offset + 1] == b'\n' {
505 self.buf_left_shift(2);
506 } else {
507 self.buf_left_shift(1);
508 }
509
510 return Ok(Some(c));
511 },
512 _ => (),
513 }
514
515 self.buf_left_shift(1);
516
517 c += 1;
518 },
519 _ => {
520 while self.buf_length < width {
521 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
522 {
523 Ok(0) => {
524 self.buf_left_shift(self.buf_length);
525 c += self.buf_length;
526
527 return Ok(Some(c));
528 },
529 Ok(c) => self.buf_length += c,
530 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
531 Err(err) => return Err(err.into()),
532 }
533 }
534
535 self.buf_left_shift(width);
536 c += width;
537 },
538 }
539
540 if self.buf_length == 0 {
541 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
542
543 if size == 0 {
544 return Ok(Some(c));
545 }
546
547 self.buf_length += size;
548 }
549 }
550 }
551}
552
553impl<R: Read, const N: usize> Scanner<R, N> {
554 /// Skip the next whitespaces (`javaWhitespace`). If there is nothing to read, it will return `Ok(false)`.
555 ///
556 /// ```rust
557 /// use scanner_rust::Scanner;
558 ///
559 /// let mut sc = Scanner::new("1 2 c".as_bytes());
560 ///
561 /// assert_eq!(Some('1'), sc.next_char().unwrap());
562 /// assert_eq!(Some(' '), sc.next_char().unwrap());
563 /// assert_eq!(Some('2'), sc.next_char().unwrap());
564 /// assert_eq!(true, sc.skip_whitespaces().unwrap());
565 /// assert_eq!(Some('c'), sc.next_char().unwrap());
566 /// assert_eq!(false, sc.skip_whitespaces().unwrap());
567 /// ```
568 pub fn skip_whitespaces(&mut self) -> Result<bool, ScannerError> {
569 if !self.passing_read()? {
570 return Ok(false);
571 }
572
573 loop {
574 let e = self.buf[self.buf_offset];
575
576 let width = get_width(e);
577
578 match width {
579 0 => {
580 break;
581 },
582 1 => {
583 if !is_whitespace_1(e) {
584 break;
585 }
586
587 self.buf_left_shift(1);
588 },
589 3 => {
590 while self.buf_length < width {
591 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
592 {
593 Ok(0) => {
594 return Ok(true);
595 },
596 Ok(c) => self.buf_length += c,
597 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
598 Err(err) => return Err(err.into()),
599 }
600 }
601
602 if is_whitespace_3(
603 self.buf[self.buf_offset],
604 self.buf[self.buf_offset + 1],
605 self.buf[self.buf_offset + 2],
606 ) {
607 self.buf_left_shift(3);
608 } else {
609 break;
610 }
611 },
612 _ => {
613 break;
614 },
615 }
616
617 if self.buf_length == 0 {
618 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
619
620 if size == 0 {
621 return Ok(true);
622 }
623
624 self.buf_length += size;
625 }
626 }
627
628 Ok(true)
629 }
630
631 /// Read the next token separated by whitespaces. If there is nothing to read, it will return `Ok(None)`.
632 ///
633 /// ```rust
634 /// use scanner_rust::Scanner;
635 ///
636 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
637 ///
638 /// assert_eq!(Some("123".into()), sc.next().unwrap());
639 /// assert_eq!(Some("456".into()), sc.next().unwrap());
640 /// assert_eq!(Some("789".into()), sc.next().unwrap());
641 /// assert_eq!(Some("中文".into()), sc.next().unwrap());
642 /// assert_eq!(None, sc.next().unwrap());
643 /// ```
644 #[allow(clippy::should_implement_trait)]
645 pub fn next(&mut self) -> Result<Option<String>, ScannerError> {
646 if !self.skip_whitespaces()? {
647 return Ok(None);
648 }
649
650 if self.buf_length == 0 {
651 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
652
653 if size == 0 {
654 return Ok(None);
655 }
656
657 self.buf_length += size;
658 }
659
660 let mut temp = String::new();
661
662 loop {
663 let e = self.buf[self.buf_offset];
664
665 let width = get_width(e);
666
667 match width {
668 0 => {
669 self.buf_left_shift(1);
670
671 temp.push(REPLACEMENT_CHARACTER);
672 },
673 1 => {
674 if is_whitespace_1(e) {
675 return Ok(Some(temp));
676 }
677
678 self.buf_left_shift(1);
679
680 temp.push(e as char);
681 },
682 3 => {
683 while self.buf_length < width {
684 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
685 {
686 Ok(0) => {
687 temp.push_str(
688 String::from_utf8_lossy(
689 &self.buf
690 [self.buf_offset..(self.buf_offset + self.buf_length)],
691 )
692 .as_ref(),
693 );
694
695 self.buf_left_shift(self.buf_length);
696
697 return Ok(Some(temp));
698 },
699 Ok(c) => self.buf_length += c,
700 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
701 Err(err) => return Err(err.into()),
702 }
703 }
704
705 if is_whitespace_3(
706 self.buf[self.buf_offset],
707 self.buf[self.buf_offset + 1],
708 self.buf[self.buf_offset + 2],
709 ) {
710 return Ok(Some(temp));
711 } else {
712 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
713
714 match from_utf8(char_str_bytes) {
715 Ok(char_str) => {
716 temp.push_str(char_str);
717
718 self.buf_left_shift(width);
719 },
720 Err(_) => {
721 self.buf_left_shift(1);
722
723 temp.push(REPLACEMENT_CHARACTER);
724 },
725 }
726 }
727 },
728 _ => {
729 while self.buf_length < width {
730 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
731 {
732 Ok(0) => {
733 temp.push_str(
734 String::from_utf8_lossy(
735 &self.buf
736 [self.buf_offset..(self.buf_offset + self.buf_length)],
737 )
738 .as_ref(),
739 );
740
741 self.buf_left_shift(self.buf_length);
742
743 return Ok(Some(temp));
744 },
745 Ok(c) => self.buf_length += c,
746 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
747 Err(err) => return Err(err.into()),
748 }
749 }
750
751 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
752
753 match from_utf8(char_str_bytes) {
754 Ok(char_str) => {
755 temp.push_str(char_str);
756
757 self.buf_left_shift(width);
758 },
759 Err(_) => {
760 self.buf_left_shift(1);
761
762 temp.push(REPLACEMENT_CHARACTER);
763 },
764 }
765 },
766 }
767
768 if self.buf_length == 0 {
769 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
770
771 if size == 0 {
772 return Ok(Some(temp));
773 }
774
775 self.buf_length += size;
776 }
777 }
778 }
779
780 /// Read the next token separated by whitespaces without fully validating UTF-8. If there is nothing to read, it will return `Ok(None)`.
781 ///
782 /// ```rust
783 /// use scanner_rust::Scanner;
784 ///
785 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
786 ///
787 /// assert_eq!(Some("123".into()), sc.next_raw().unwrap());
788 /// assert_eq!(Some("456".into()), sc.next_raw().unwrap());
789 /// assert_eq!(Some("789".into()), sc.next_raw().unwrap());
790 /// assert_eq!(Some("中文".into()), sc.next_raw().unwrap());
791 /// assert_eq!(None, sc.next_raw().unwrap());
792 /// ```
793 pub fn next_raw(&mut self) -> Result<Option<Vec<u8>>, ScannerError> {
794 if !self.skip_whitespaces()? {
795 return Ok(None);
796 }
797
798 if self.buf_length == 0 {
799 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
800
801 if size == 0 {
802 return Ok(None);
803 }
804
805 self.buf_length += size;
806 }
807
808 let mut temp = Vec::new();
809
810 loop {
811 let e = self.buf[self.buf_offset];
812
813 let width = get_width(e);
814
815 match width {
816 0 => {
817 self.buf_left_shift(1);
818
819 temp.push(e);
820 },
821 1 => {
822 if is_whitespace_1(e) {
823 return Ok(Some(temp));
824 }
825
826 self.buf_left_shift(1);
827
828 temp.push(e);
829 },
830 3 => {
831 while self.buf_length < width {
832 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
833 {
834 Ok(0) => {
835 self.buf_left_shift(self.buf_length);
836
837 return Ok(Some(temp));
838 },
839 Ok(c) => self.buf_length += c,
840 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
841 Err(err) => return Err(err.into()),
842 }
843 }
844
845 if is_whitespace_3(
846 self.buf[self.buf_offset],
847 self.buf[self.buf_offset + 1],
848 self.buf[self.buf_offset + 2],
849 ) {
850 return Ok(Some(temp));
851 } else {
852 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
853
854 temp.extend_from_slice(char_str_bytes);
855
856 self.buf_left_shift(width);
857 }
858 },
859 _ => {
860 while self.buf_length < width {
861 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
862 {
863 Ok(0) => {
864 temp.extend_from_slice(
865 &self.buf[self.buf_offset..(self.buf_offset + self.buf_length)],
866 );
867
868 self.buf_left_shift(self.buf_length);
869
870 return Ok(Some(temp));
871 },
872 Ok(c) => self.buf_length += c,
873 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
874 Err(err) => return Err(err.into()),
875 }
876 }
877
878 let char_str_bytes = &self.buf[self.buf_offset..(self.buf_offset + width)];
879
880 temp.extend_from_slice(char_str_bytes);
881
882 self.buf_left_shift(width);
883 },
884 }
885
886 if self.buf_length == 0 {
887 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
888
889 if size == 0 {
890 return Ok(Some(temp));
891 }
892
893 self.buf_length += size;
894 }
895 }
896 }
897
898 /// Drop the next token separated by whitespaces. If there is nothing to read, it will return `Ok(None)`. If there is something to read, it will return `Ok(Some(i))`. The `i` is the length of the dropped token.
899 ///
900 /// ```rust
901 /// use scanner_rust::Scanner;
902 ///
903 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
904 ///
905 /// assert_eq!(Some(3), sc.drop_next().unwrap());
906 /// assert_eq!(Some("456".into()), sc.next().unwrap());
907 /// assert_eq!(Some(3), sc.drop_next().unwrap());
908 /// assert_eq!(Some("中文".into()), sc.next().unwrap());
909 /// assert_eq!(None, sc.drop_next().unwrap());
910 /// ```
911 pub fn drop_next(&mut self) -> Result<Option<usize>, ScannerError> {
912 if !self.skip_whitespaces()? {
913 return Ok(None);
914 }
915
916 if self.buf_length == 0 {
917 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
918
919 if size == 0 {
920 return Ok(None);
921 }
922
923 self.buf_length += size;
924 }
925
926 let mut c = 0;
927
928 loop {
929 let e = self.buf[self.buf_offset];
930
931 let width = get_width(e);
932
933 match width {
934 0 => {
935 self.buf_left_shift(1);
936
937 c += 1;
938 },
939 1 => {
940 if is_whitespace_1(e) {
941 return Ok(Some(c));
942 }
943
944 self.buf_left_shift(1);
945
946 c += 1;
947 },
948 3 => {
949 while self.buf_length < width {
950 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
951 {
952 Ok(0) => {
953 self.buf_left_shift(self.buf_length);
954 c += self.buf_length;
955
956 return Ok(Some(c));
957 },
958 Ok(c) => self.buf_length += c,
959 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
960 Err(err) => return Err(err.into()),
961 }
962 }
963
964 if is_whitespace_3(
965 self.buf[self.buf_offset],
966 self.buf[self.buf_offset + 1],
967 self.buf[self.buf_offset + 2],
968 ) {
969 return Ok(Some(c));
970 } else {
971 self.buf_left_shift(width);
972 }
973 },
974 _ => {
975 while self.buf_length < width {
976 match self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])
977 {
978 Ok(0) => {
979 self.buf_left_shift(self.buf_length);
980 c += self.buf_length;
981
982 return Ok(Some(c));
983 },
984 Ok(c) => self.buf_length += c,
985 Err(ref err) if err.kind() == ErrorKind::Interrupted => (),
986 Err(err) => return Err(err.into()),
987 }
988 }
989
990 self.buf_left_shift(width);
991 },
992 }
993
994 if self.buf_length == 0 {
995 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
996
997 if size == 0 {
998 return Ok(Some(c));
999 }
1000
1001 self.buf_length += size;
1002 }
1003 }
1004 }
1005}
1006
1007impl<R: Read, const N: usize> Scanner<R, N> {
1008 /// Read the next bytes. If there is nothing to read, it will return `Ok(None)`.
1009 ///
1010 /// ```rust
1011 /// use scanner_rust::Scanner;
1012 ///
1013 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
1014 ///
1015 /// assert_eq!(Some("123".into()), sc.next_bytes(3).unwrap());
1016 /// assert_eq!(Some(" 456".into()), sc.next_bytes(4).unwrap());
1017 /// assert_eq!(Some("\r\n789 ".into()), sc.next_bytes(6).unwrap());
1018 /// assert_eq!(Some("中文".into()), sc.next_raw().unwrap());
1019 /// assert_eq!(Some(" ".into()), sc.next_bytes(2).unwrap());
1020 /// assert_eq!(None, sc.next_bytes(2).unwrap());
1021 /// ```
1022 pub fn next_bytes(
1023 &mut self,
1024 max_number_of_bytes: usize,
1025 ) -> Result<Option<Vec<u8>>, ScannerError> {
1026 if !self.passing_read()? {
1027 return Ok(None);
1028 }
1029
1030 let mut temp = Vec::new();
1031 let mut c = 0;
1032
1033 while c < max_number_of_bytes {
1034 if self.buf_length == 0 {
1035 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
1036
1037 if size == 0 {
1038 return Ok(Some(temp));
1039 }
1040
1041 self.buf_length += size;
1042 }
1043
1044 let dropping_bytes = self.buf_length.min(max_number_of_bytes - c);
1045
1046 temp.extend_from_slice(&self.buf[self.buf_offset..(self.buf_offset + dropping_bytes)]);
1047
1048 self.buf_left_shift(dropping_bytes);
1049
1050 c += dropping_bytes;
1051 }
1052
1053 Ok(Some(temp))
1054 }
1055
1056 /// Drop the next N bytes. If there is nothing to read, it will return `Ok(None)`. If there is something to read, it will return `Ok(Some(i))`. The `i` is the length of the actually dropped bytes.
1057 ///
1058 /// ```rust
1059 /// use scanner_rust::Scanner;
1060 ///
1061 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
1062 ///
1063 /// assert_eq!(Some(7), sc.drop_next_bytes(7).unwrap());
1064 /// assert_eq!(Some("".into()), sc.next_line().unwrap());
1065 /// assert_eq!(Some("789 ".into()), sc.next_line().unwrap());
1066 /// assert_eq!(Some(1), sc.drop_next_bytes(1).unwrap());
1067 /// assert_eq!(Some(" 中文 ".into()), sc.next_line().unwrap());
1068 /// assert_eq!(None, sc.drop_next_bytes(1).unwrap());
1069 /// ```
1070 pub fn drop_next_bytes(
1071 &mut self,
1072 max_number_of_bytes: usize,
1073 ) -> Result<Option<usize>, ScannerError> {
1074 if !self.passing_read()? {
1075 return Ok(None);
1076 }
1077
1078 let mut c = 0;
1079
1080 while c < max_number_of_bytes {
1081 if self.buf_length == 0 {
1082 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
1083
1084 if size == 0 {
1085 return Ok(Some(c));
1086 }
1087
1088 self.buf_length += size;
1089 }
1090
1091 let dropping_bytes = self.buf_length.min(max_number_of_bytes - c);
1092
1093 self.buf_left_shift(dropping_bytes);
1094
1095 c += dropping_bytes;
1096 }
1097
1098 Ok(Some(c))
1099 }
1100}
1101
1102impl<R: Read, const N: usize> Scanner<R, N> {
1103 /// Read the next text until it reaches a specific boundary. If there is nothing to read, it will return `Ok(None)`.
1104 ///
1105 /// ```rust
1106 /// use scanner_rust::Scanner;
1107 ///
1108 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
1109 ///
1110 /// assert_eq!(Some("123".into()), sc.next_until(" ").unwrap());
1111 /// assert_eq!(Some("456\r".into()), sc.next_until("\n").unwrap());
1112 /// assert_eq!(Some("78".into()), sc.next_until("9 ").unwrap());
1113 /// assert_eq!(Some("\n\n 中文 ".into()), sc.next_until("kk").unwrap());
1114 /// assert_eq!(None, sc.next().unwrap());
1115 /// ```
1116 pub fn next_until<S: AsRef<str>>(
1117 &mut self,
1118 boundary: S,
1119 ) -> Result<Option<String>, ScannerError> {
1120 let boundary = boundary.as_ref();
1121
1122 Ok(self
1123 .next_until_raw(boundary.as_bytes())?
1124 .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()))
1125 }
1126
1127 /// Read the next data until it reaches a specific boundary without fully validating UTF-8. If there is nothing to read, it will return `Ok(None)`.
1128 ///
1129 /// ```rust
1130 /// use scanner_rust::Scanner;
1131 ///
1132 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
1133 ///
1134 /// assert_eq!(Some("123".into()), sc.next_until_raw(" ").unwrap());
1135 /// assert_eq!(Some("456\r".into()), sc.next_until_raw("\n").unwrap());
1136 /// assert_eq!(Some("78".into()), sc.next_until_raw("9 ").unwrap());
1137 /// assert_eq!(Some("\n\n 中文 ".into()), sc.next_until_raw("kk").unwrap());
1138 /// assert_eq!(None, sc.next().unwrap());
1139 /// ```
1140 pub fn next_until_raw<D: ?Sized + AsRef<[u8]>>(
1141 &mut self,
1142 boundary: &D,
1143 ) -> Result<Option<Vec<u8>>, ScannerError> {
1144 if !self.passing_read()? {
1145 return Ok(None);
1146 }
1147
1148 let boundary = boundary.as_ref();
1149 let boundary_length = boundary.len();
1150 let mut temp = Vec::new();
1151
1152 // An empty boundary never matches, so keep reading until the end.
1153 if boundary_length == 0 {
1154 loop {
1155 temp.extend_from_slice(
1156 &self.buf[self.buf_offset..(self.buf_offset + self.buf_length)],
1157 );
1158
1159 self.buf_left_shift(self.buf_length);
1160
1161 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
1162
1163 if size == 0 {
1164 return Ok(Some(temp));
1165 }
1166
1167 self.buf_length += size;
1168 }
1169 }
1170
1171 // Match the boundary across buffer refills with KMP so a mismatch never re-reads bytes.
1172 let lps = compute_lps(boundary);
1173 let mut b = 0;
1174
1175 loop {
1176 let mut i = 0;
1177
1178 while i < self.buf_length {
1179 let e = self.buf[self.buf_offset + i];
1180
1181 while b > 0 && e != boundary[b] {
1182 b = lps[b - 1];
1183 }
1184
1185 if e == boundary[b] {
1186 b += 1;
1187 }
1188
1189 i += 1;
1190
1191 if b == boundary_length {
1192 // The matched boundary is the last `boundary_length` bytes seen so far.
1193 temp.extend_from_slice(&self.buf[self.buf_offset..(self.buf_offset + i)]);
1194 temp.truncate(temp.len() - boundary_length);
1195
1196 self.buf_left_shift(i);
1197
1198 return Ok(Some(temp));
1199 }
1200 }
1201
1202 temp.extend_from_slice(&self.buf[self.buf_offset..(self.buf_offset + self.buf_length)]);
1203
1204 self.buf_left_shift(self.buf_length);
1205
1206 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
1207
1208 if size == 0 {
1209 return Ok(Some(temp));
1210 }
1211
1212 self.buf_length += size;
1213 }
1214 }
1215
1216 /// Drop the next data until it reaches a specific boundary. If there is nothing to read, it will return `Ok(None)`.
1217 ///
1218 /// ```rust
1219 /// use scanner_rust::Scanner;
1220 ///
1221 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
1222 ///
1223 /// assert_eq!(Some(7), sc.drop_next_until("\r\n").unwrap());
1224 /// assert_eq!(Some("789 ".into()), sc.next_line().unwrap());
1225 /// assert_eq!(Some(0), sc.drop_next_until("\n").unwrap());
1226 /// assert_eq!(Some(" 中文 ".into()), sc.next_line().unwrap());
1227 /// assert_eq!(None, sc.drop_next_until("").unwrap());
1228 /// ```
1229 pub fn drop_next_until<D: ?Sized + AsRef<[u8]>>(
1230 &mut self,
1231 boundary: &D,
1232 ) -> Result<Option<usize>, ScannerError> {
1233 if !self.passing_read()? {
1234 return Ok(None);
1235 }
1236
1237 let boundary = boundary.as_ref();
1238 let boundary_length = boundary.len();
1239 let mut c = 0;
1240
1241 // An empty boundary never matches, so keep reading until the end.
1242 if boundary_length == 0 {
1243 loop {
1244 c += self.buf_length;
1245
1246 self.buf_left_shift(self.buf_length);
1247
1248 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
1249
1250 if size == 0 {
1251 return Ok(Some(c));
1252 }
1253
1254 self.buf_length += size;
1255 }
1256 }
1257
1258 // Match the boundary across buffer refills with KMP so a mismatch never re-reads bytes.
1259 let lps = compute_lps(boundary);
1260 let mut b = 0;
1261
1262 loop {
1263 let mut i = 0;
1264
1265 while i < self.buf_length {
1266 let e = self.buf[self.buf_offset + i];
1267
1268 while b > 0 && e != boundary[b] {
1269 b = lps[b - 1];
1270 }
1271
1272 if e == boundary[b] {
1273 b += 1;
1274 }
1275
1276 i += 1;
1277
1278 if b == boundary_length {
1279 self.buf_left_shift(i);
1280
1281 return Ok(Some(c + i - boundary_length));
1282 }
1283 }
1284
1285 c += self.buf_length;
1286
1287 self.buf_left_shift(self.buf_length);
1288
1289 let size = self.reader.read(&mut self.buf[self.buf_offset..])?;
1290
1291 if size == 0 {
1292 return Ok(Some(c));
1293 }
1294
1295 self.buf_length += size;
1296 }
1297 }
1298}
1299
1300impl<R: Read, const N: usize> Scanner<R, N> {
1301 /// Fill up the buffer as much as possible and return an immutable slice of all the currently buffered (unread) data.
1302 /// Reading stops at the end of the stream or when the buffer is full. If `shift` is `true`, the buffered data is first moved to the front of the buffer so that up to the whole buffer size can be filled; if `false`, only the space after the current read position is filled.
1303 ///
1304 /// ```rust
1305 /// use scanner_rust::Scanner;
1306 ///
1307 /// let mut sc = Scanner::new("123 456\r\n789 \n\n 中文 ".as_bytes());
1308 ///
1309 /// assert_eq!("123 456\r\n789 \n\n 中文 ".as_bytes(), sc.peek(false).unwrap());
1310 /// ```
1311 #[inline]
1312 pub fn peek(&mut self, shift: bool) -> Result<&[u8], ScannerError> {
1313 // Consume any line-terminator byte deferred by a previous read so it is not peeked again.
1314 self.passing_read()?;
1315
1316 if shift {
1317 self.buf_align_to_front_end();
1318 }
1319
1320 loop {
1321 let size = self.reader.read(&mut self.buf[(self.buf_offset + self.buf_length)..])?;
1322
1323 if size == 0 {
1324 break;
1325 }
1326
1327 self.buf_length += size;
1328 }
1329
1330 Ok(&self.buf[self.buf_offset..(self.buf_offset + self.buf_length)])
1331 }
1332}
1333
1334impl<R: Read, const N: usize> Scanner<R, N> {
1335 #[inline]
1336 fn next_raw_parse<T: FromStr>(&mut self) -> Result<Option<T>, ScannerError>
1337 where
1338 ScannerError: From<<T as FromStr>::Err>, {
1339 let result = self.next_raw()?;
1340
1341 match result {
1342 // SAFETY: for malformed input `s` may not be valid UTF-8 (technically UB to treat as a `&str`), but it is only fed to a primitive `FromStr` that reads it as bytes, so an invalid token merely fails to parse; validation is skipped for speed.
1343 Some(s) => Ok(Some(unsafe { from_utf8_unchecked(&s) }.parse()?)),
1344 None => Ok(None),
1345 }
1346 }
1347}
1348
1349impl<R: Read, const N: usize> Scanner<R, N> {
1350 #[inline]
1351 fn next_until_raw_parse<T: FromStr, D: ?Sized + AsRef<[u8]>>(
1352 &mut self,
1353 boundary: &D,
1354 ) -> Result<Option<T>, ScannerError>
1355 where
1356 ScannerError: From<<T as FromStr>::Err>, {
1357 let result = self.next_until_raw(boundary)?;
1358
1359 match result {
1360 // SAFETY: for malformed input `s` may not be valid UTF-8 (technically UB to treat as a `&str`), but it is only fed to a primitive `FromStr` that reads it as bytes, so an invalid token merely fails to parse; validation is skipped for speed.
1361 Some(s) => Ok(Some(unsafe { from_utf8_unchecked(&s) }.parse()?)),
1362 None => Ok(None),
1363 }
1364 }
1365}
1366
1367macro_rules! scanner_number_methods {
1368 ($(($t:ty, $next:ident, $next_until:ident, $sample:literal, $v1:literal, $v2:literal)),+ $(,)?) => {
1369 impl<R: Read, const N: usize> Scanner<R, N> {
1370 $(
1371 #[doc = concat!(
1372 "Read the next token separated by whitespaces and parse it to a `", stringify!($t), "` value. If there is nothing to read, it will return `Ok(None)`.\n\n```rust\nuse scanner_rust::Scanner;\n\nlet mut sc = Scanner::new(", stringify!($sample), ".as_bytes());\n\nassert_eq!(Some(", stringify!($v1), "), sc.", stringify!($next), "().unwrap());\nassert_eq!(Some(", stringify!($v2), "), sc.", stringify!($next), "().unwrap());\n```"
1373 )]
1374 #[inline]
1375 pub fn $next(&mut self) -> Result<Option<$t>, ScannerError> {
1376 self.next_raw_parse()
1377 }
1378 )+
1379 }
1380
1381 impl<R: Read, const N: usize> Scanner<R, N> {
1382 $(
1383 #[doc = concat!(
1384 "Read the next text until it reaches a specific boundary and parse it to a `", stringify!($t), "` value. If there is nothing to read, it will return `Ok(None)`.\n\n```rust\nuse scanner_rust::Scanner;\n\nlet mut sc = Scanner::new(", stringify!($sample), ".as_bytes());\n\nassert_eq!(Some(", stringify!($v1), "), sc.", stringify!($next_until), "(\" \").unwrap());\nassert_eq!(Some(", stringify!($v2), "), sc.", stringify!($next_until), "(\" \").unwrap());\n```"
1385 )]
1386 #[inline]
1387 pub fn $next_until<D: ?Sized + AsRef<[u8]>>(
1388 &mut self,
1389 boundary: &D,
1390 ) -> Result<Option<$t>, ScannerError> {
1391 self.next_until_raw_parse(boundary)
1392 }
1393 )+
1394 }
1395 };
1396}
1397
1398scanner_number_methods! {
1399 (u8, next_u8, next_u8_until, "1 2", 1, 2),
1400 (u16, next_u16, next_u16_until, "1 2", 1, 2),
1401 (u32, next_u32, next_u32_until, "1 2", 1, 2),
1402 (u64, next_u64, next_u64_until, "1 2", 1, 2),
1403 (u128, next_u128, next_u128_until, "1 2", 1, 2),
1404 (usize, next_usize, next_usize_until, "1 2", 1, 2),
1405 (i8, next_i8, next_i8_until, "1 2", 1, 2),
1406 (i16, next_i16, next_i16_until, "1 2", 1, 2),
1407 (i32, next_i32, next_i32_until, "1 2", 1, 2),
1408 (i64, next_i64, next_i64_until, "1 2", 1, 2),
1409 (i128, next_i128, next_i128_until, "1 2", 1, 2),
1410 (isize, next_isize, next_isize_until, "1 2", 1, 2),
1411 (f32, next_f32, next_f32_until, "1 2.5", 1.0, 2.5),
1412 (f64, next_f64, next_f64_until, "1 2.5", 1.0, 2.5),
1413}