1#[cfg(feature = "alloc")]
28use alloc::borrow::Cow;
29#[cfg(feature = "alloc")]
30use alloc::string::String;
31#[cfg(feature = "alloc")]
32use alloc::vec::Vec;
33
34use super::DecoderResult;
35use super::in_inclusive_range8;
36use super::in_inclusive_range16;
37use super::in_inclusive_range32;
38use super::in_range16;
39use super::in_range32;
40use crate::ascii::*;
41use crate::utf_8::*;
42
43macro_rules! non_fuzz_debug_assert {
44 ($($arg:tt)*) => (if !cfg!(fuzzing) { debug_assert!($($arg)*); })
45}
46
47cfg_if! {
48 if #[cfg(feature = "simd-accel")] {
49 use ::core::intrinsics::likely;
50 use ::core::intrinsics::unlikely;
51 } else {
52 #[inline(always)]
53 fn likely(b: bool) -> bool {
54 b
55 }
56 #[inline(always)]
57 fn unlikely(b: bool) -> bool {
58 b
59 }
60 }
61}
62
63#[must_use]
67#[derive(Debug, PartialEq, Eq)]
68#[repr(C)]
69pub enum Latin1Bidi {
70 Latin1 = 0,
72 LeftToRight = 1,
75 Bidi = 2,
77}
78
79#[inline(always)]
80#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
81fn is_utf8_latin1_impl(buffer: &[u8]) -> Option<usize> {
82 let mut bytes = buffer;
83 let mut total = 0;
84 loop {
85 if let Some((byte, offset)) = validate_ascii(bytes) {
86 total += offset;
87 if in_inclusive_range8(byte, 0xC2, 0xC3) {
88 let next = offset + 1;
89 if next == bytes.len() {
90 return Some(total);
91 }
92 if bytes[next] & 0xC0 != 0x80 {
93 return Some(total);
94 }
95 bytes = &bytes[offset + 2..];
96 total += 2;
97 } else {
98 return Some(total);
99 }
100 } else {
101 return None;
102 }
103 }
104}
105
106macro_rules! copy_impl {
107 ($name:ident, $stride:ident, $src_unit:ty, $dst_unit:ty) => {
108 #[inline(always)]
109 fn $name(src: &[$src_unit], dst: &mut [$dst_unit]) {
110 let len = core::cmp::min(src.len(), dst.len());
112 let (src_strides, src_tail) = src[..len].as_chunks::<STRIDE>();
113 let (dst_strides, dst_tail) = dst[..len].as_chunks_mut::<STRIDE>();
114 let (src_double_strides, src_single_stride) = src_strides.as_chunks::<2>();
115 let (dst_double_strides, dst_single_stride) = dst_strides.as_chunks_mut::<2>();
116 for (src_double_stride, dst_double_stride) in
117 src_double_strides.iter().zip(dst_double_strides.iter_mut())
118 {
119 $stride(&src_double_stride[0], &mut dst_double_stride[0]);
120 $stride(&src_double_stride[1], &mut dst_double_stride[1]);
121 }
122 for (src_stride, dst_stride) in
123 src_single_stride.iter().zip(dst_single_stride.iter_mut())
124 {
125 $stride(src_stride, dst_stride);
126 }
127 for (src_slot, dst_slot) in src_tail.iter().zip(dst_tail.iter_mut()) {
128 *dst_slot = *src_slot as $dst_unit;
129 }
130 }
131 };
132}
133
134copy_impl!(unpack_latin1, unpack_stride, u8, u16);
135copy_impl!(pack_latin1, pack_stride, u16, u8);
136
137cfg_if! {
138 if #[cfg(all(
139 feature = "simd-accel",
140 target_endian = "little",
141 ))] {
142 use core::simd::u8x16;
143 use core::simd::u16x8;
144
145 use crate::simd_funcs::unpack_stride;
146 use crate::simd_funcs::pack_stride;
147
148 #[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
149 fn is_utf16_bidi_impl(buffer: &[u16]) -> bool {
150 let (half_strides, tail) = buffer.as_chunks::<{STRIDE / 2}>();
151 for half_stride in half_strides {
152 if crate::simd_funcs::is_half_stride_bidi(half_stride) {
153 return true;
154 }
155 }
156 tail.iter().any(|c| is_utf16_code_unit_bidi(*c))
157 }
158
159 #[inline(always)]
160 fn is_str_latin1_bool_impl(buffer: &str) -> bool {
161 let (strides, tail) = buffer.as_bytes().as_chunks::<STRIDE>();
162 for stride in strides {
163 let simd = (*stride).into();
164 if !crate::simd_funcs::simd_is_str_latin1(simd) {
165 return false;
166 }
167 }
168 tail.iter().all(|b| *b < 0xC4)
169 }
170
171 macro_rules! unit_check_impl {
172 ($name:ident, $stride:ident, $unit:ty, $simd:ty, $bound:expr) => {
173 #[inline(always)]
174 fn $name(buffer: &[$unit]) -> bool {
175 if let Some(u) = buffer.first() {
178 if *u >= $bound {
179 return false;
180 }
181 }
182 let (strides, tail) = buffer.as_chunks::<{STRIDE / core::mem::size_of::<$unit>()}>();
183 let (quad_strides, strides_tail) = strides.as_chunks::<4>();
184 for quad_stride in quad_strides {
185 if let Some(reduced) = quad_stride.iter().map(|s| { let simd: $simd = (*s).into(); simd }).reduce(|a, b| a | b) {
186 if !crate::simd_funcs::$stride(reduced) {
187 return false;
188 }
189 } else {
190 debug_assert!(false);
191 }
192 }
193 if let Some(reduced) = strides_tail.iter().map(|s| { let simd: $simd = (*s).into(); simd }).reduce(|a, b| a | b) {
194 if !crate::simd_funcs::$stride(reduced) {
195 return false;
196 }
197 }
198 if let Some(reduced) = tail.iter().copied().reduce(|a, b| a | b) {
199 reduced < $bound
200 } else {
201 true
202 }
203 }
204 };
205 }
206
207 unit_check_impl!(is_ascii_impl, simd_is_ascii, u8, u8x16, 0x80);
208 unit_check_impl!(is_basic_latin_impl, simd_is_basic_latin, u16, u16x8, 0x80);
209 unit_check_impl!(is_utf16_latin1_impl, simd_is_latin1, u16, u16x8, 0x100);
210
211 #[inline(always)]
212 fn check_utf16_for_latin1_and_bidi_impl(buffer: &[u16]) -> Latin1Bidi {
213 let (half_strides, tail) = buffer.as_chunks::<{STRIDE / 2}>();
214 let mut half_stride_iter = half_strides.iter();
215 loop {
216 let Some(s) = half_stride_iter.next() else {
217 let mut iter = tail.iter();
218 loop {
219 let Some(u) = iter.next() else {
220 return Latin1Bidi::Latin1;
221 };
222 if *u < 0x100 {
223 continue;
224 }
225 if is_utf16_code_unit_bidi(*u) {
226 return Latin1Bidi::Bidi;
227 }
228 loop {
229 let Some(u) = iter.next() else {
230 return Latin1Bidi::LeftToRight;
231 };
232 if is_utf16_code_unit_bidi(*u) {
233 return Latin1Bidi::Bidi;
234 }
235 }
236 }
237 };
238 let simd: u16x8 = (*s).into();
239 if crate::simd_funcs::simd_is_latin1(simd) {
240 continue;
241 }
242 if crate::simd_funcs::is_u16x8_bidi(simd) {
243 return Latin1Bidi::Bidi;
244 }
245 loop {
246 let Some(s) = half_stride_iter.next() else {
247 for u in tail {
248 if is_utf16_code_unit_bidi(*u) {
249 return Latin1Bidi::Bidi;
250 }
251 }
252 return Latin1Bidi::LeftToRight;
253 };
254 let simd: u16x8 = (*s).into();
255 if crate::simd_funcs::is_u16x8_bidi(simd) {
256 return Latin1Bidi::Bidi;
257 }
258 }
259 }
260 }
261
262 } else {
263 use crate::ascii::unpack_stride;
264 use crate::ascii::pack_stride;
265
266 fn is_utf16_bidi_impl(buffer: &[u16]) -> bool {
267 buffer.iter().any(|c| is_utf16_code_unit_bidi(*c))
268 }
269
270 #[inline(always)]
271 fn is_str_latin1_bool_impl(buffer: &str) -> bool {
272 is_str_latin1_impl(buffer).is_none()
273 }
274
275 macro_rules! unit_check_impl {
276 ($name:ident, $stride:ident, $unit:ty, $bound:expr) => {
277 #[inline(always)]
278 fn $name(buffer: &[$unit]) -> bool {
279 let (strides, tail) = buffer.as_chunks::<STRIDE>();
280 for stride in strides {
281 if !crate::ascii::$stride(stride) {
282 return false;
283 }
284 }
285 if let Some(reduced) = tail.iter().copied().reduce(|a, b| a | b) {
286 reduced < $bound
287 } else {
288 true
289 }
290 }
291 };
292 }
293
294 unit_check_impl!(is_ascii_impl, is_ascii, u8, 0x80);
295 unit_check_impl!(is_basic_latin_impl, is_basic_latin, u16, 0x80);
296 unit_check_impl!(is_utf16_latin1_impl, is_utf16_latin1, u16, 0x100);
297
298 #[inline(always)]
299 fn check_utf16_for_latin1_and_bidi_impl(buffer: &[u16]) -> Latin1Bidi {
300 let mut iter = buffer.iter();
301 loop {
302 let Some(u) = iter.next() else {
303 return Latin1Bidi::Latin1;
304 };
305 if *u < 0x100 {
306 continue;
307 }
308 if is_utf16_code_unit_bidi(*u) {
309 return Latin1Bidi::Bidi;
310 }
311 loop {
312 let Some(u) = iter.next() else {
313 return Latin1Bidi::LeftToRight;
314 };
315 if is_utf16_code_unit_bidi(*u) {
316 return Latin1Bidi::Bidi;
317 }
318 }
319 }
320 }
321
322 }
323}
324
325pub fn is_ascii(buffer: &[u8]) -> bool {
330 is_ascii_impl(buffer)
331}
332
333#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
339pub fn is_basic_latin(buffer: &[u16]) -> bool {
340 is_basic_latin_impl(buffer)
341}
342
343pub fn is_utf8_latin1(buffer: &[u8]) -> bool {
349 is_utf8_latin1_impl(buffer).is_none()
350}
351
352pub fn is_str_latin1(buffer: &str) -> bool {
358 is_str_latin1_bool_impl(buffer)
359}
360
361#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
367pub fn is_utf16_latin1(buffer: &[u16]) -> bool {
368 is_utf16_latin1_impl(buffer)
369}
370
371#[allow(clippy::collapsible_if, clippy::cognitive_complexity)]
388#[inline]
389#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
390pub fn is_utf8_bidi(buffer: &[u8]) -> bool {
391 let mut src = buffer;
430 'outer: loop {
431 if let Some((mut byte, mut read)) = validate_ascii(src) {
432 if read + 4 <= src.len() {
435 'inner: loop {
436 match byte {
438 0..=0x7F => {
439 read += 1;
441 src = &src[read..];
442 continue 'outer;
443 }
444 0xC2..=0xD5 => {
445 let second = unsafe { *(src.get_unchecked(read + 1)) };
447 if !in_inclusive_range8(second, 0x80, 0xBF) {
448 return true;
449 }
450 read += 2;
451 }
452 0xD6 => {
453 let second = unsafe { *(src.get_unchecked(read + 1)) };
455 if !in_inclusive_range8(second, 0x80, 0xBF) {
456 return true;
457 }
458 if second > 0x8F {
460 return true;
461 }
462 read += 2;
463 }
464 0xE1 | 0xE3..=0xEC | 0xEE => {
466 let second = unsafe { *(src.get_unchecked(read + 1)) };
468 let third = unsafe { *(src.get_unchecked(read + 2)) };
469 if ((UTF8_DATA.table[usize::from(second)]
470 & unsafe {
471 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
472 })
473 | (third >> 6))
474 != 2
475 {
476 return true;
477 }
478 read += 3;
479 }
480 0xE2 => {
481 let second = unsafe { *(src.get_unchecked(read + 1)) };
483 let third = unsafe { *(src.get_unchecked(read + 2)) };
484 if ((UTF8_DATA.table[usize::from(second)]
485 & unsafe {
486 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
487 })
488 | (third >> 6))
489 != 2
490 {
491 return true;
492 }
493 if second == 0x80 {
494 if third == 0x8F || third == 0xAB || third == 0xAE {
495 return true;
496 }
497 } else if second == 0x81 {
498 if third == 0xA7 {
499 return true;
500 }
501 }
502 read += 3;
503 }
504 0xEF => {
505 let second = unsafe { *(src.get_unchecked(read + 1)) };
507 let third = unsafe { *(src.get_unchecked(read + 2)) };
508 if ((UTF8_DATA.table[usize::from(second)]
509 & unsafe {
510 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
511 })
512 | (third >> 6))
513 != 2
514 {
515 return true;
516 }
517 if in_inclusive_range8(second, 0xAC, 0xB7) {
518 if second == 0xAC {
519 if third > 0x9C {
520 return true;
521 }
522 } else {
523 return true;
524 }
525 } else if in_inclusive_range8(second, 0xB9, 0xBB) {
526 if second == 0xB9 {
527 if third > 0xAF {
528 return true;
529 }
530 } else if second == 0xBB {
531 if third != 0xBF {
532 return true;
533 }
534 } else {
535 return true;
536 }
537 }
538 read += 3;
539 }
540 0xE0 => {
541 let second = unsafe { *(src.get_unchecked(read + 1)) };
543 let third = unsafe { *(src.get_unchecked(read + 2)) };
544 if ((UTF8_DATA.table[usize::from(second)]
545 & unsafe {
546 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
547 })
548 | (third >> 6))
549 != 2
550 {
551 return true;
552 }
553 if second < 0xA4 {
555 return true;
556 }
557 read += 3;
558 }
559 0xED => {
560 let second = unsafe { *(src.get_unchecked(read + 1)) };
562 let third = unsafe { *(src.get_unchecked(read + 2)) };
563 if ((UTF8_DATA.table[usize::from(second)]
564 & unsafe {
565 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
566 })
567 | (third >> 6))
568 != 2
569 {
570 return true;
571 }
572 read += 3;
573 }
574 0xF1..=0xF4 => {
575 let second = unsafe { *(src.get_unchecked(read + 1)) };
577 let third = unsafe { *(src.get_unchecked(read + 2)) };
578 let fourth = unsafe { *(src.get_unchecked(read + 3)) };
579 if (u16::from(
580 UTF8_DATA.table[usize::from(second)]
581 & unsafe {
582 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
583 },
584 ) | u16::from(third >> 6)
585 | (u16::from(fourth & 0xC0) << 2))
586 != 0x202
587 {
588 return true;
589 }
590 read += 4;
591 }
592 0xF0 => {
593 let second = unsafe { *(src.get_unchecked(read + 1)) };
595 let third = unsafe { *(src.get_unchecked(read + 2)) };
596 let fourth = unsafe { *(src.get_unchecked(read + 3)) };
597 if (u16::from(
598 UTF8_DATA.table[usize::from(second)]
599 & unsafe {
600 *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80))
601 },
602 ) | u16::from(third >> 6)
603 | (u16::from(fourth & 0xC0) << 2))
604 != 0x202
605 {
606 return true;
607 }
608 if unlikely(second == 0x90 || second == 0x9E) {
609 let third = src[read + 2];
610 if third >= 0xA0 {
611 return true;
612 }
613 }
614 read += 4;
615 }
616 _ => {
617 return true;
619 }
620 }
621 if read + 4 > src.len() {
622 if read == src.len() {
623 return false;
624 }
625 byte = src[read];
626 break 'inner;
627 }
628 byte = src[read];
629 continue 'inner;
630 }
631 }
632 match byte {
637 0..=0x7F => {
638 read += 1;
640 src = &src[read..];
641 continue 'outer;
642 }
643 0xC2..=0xD5 => {
644 let new_read = read + 2;
646 if new_read > src.len() {
647 return true;
648 }
649 let second = unsafe { *(src.get_unchecked(read + 1)) };
650 if !in_inclusive_range8(second, 0x80, 0xBF) {
651 return true;
652 }
653 read = new_read;
654 src = &src[read..];
657 continue 'outer;
658 }
659 0xD6 => {
660 let new_read = read + 2;
662 if new_read > src.len() {
663 return true;
664 }
665 let second = unsafe { *(src.get_unchecked(read + 1)) };
666 if !in_inclusive_range8(second, 0x80, 0xBF) {
667 return true;
668 }
669 if second > 0x8F {
671 return true;
672 }
673 read = new_read;
674 src = &src[read..];
677 continue 'outer;
678 }
679 0xE1 | 0xE3..=0xEC | 0xEE => {
681 let new_read = read + 3;
683 if new_read > src.len() {
684 return true;
685 }
686 let second = unsafe { *(src.get_unchecked(read + 1)) };
687 let third = unsafe { *(src.get_unchecked(read + 2)) };
688 if ((UTF8_DATA.table[usize::from(second)]
689 & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
690 | (third >> 6))
691 != 2
692 {
693 return true;
694 }
695 }
696 0xE2 => {
697 let new_read = read + 3;
699 if new_read > src.len() {
700 return true;
701 }
702 let second = unsafe { *(src.get_unchecked(read + 1)) };
703 let third = unsafe { *(src.get_unchecked(read + 2)) };
704 if ((UTF8_DATA.table[usize::from(second)]
705 & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
706 | (third >> 6))
707 != 2
708 {
709 return true;
710 }
711 if second == 0x80 {
712 if third == 0x8F || third == 0xAB || third == 0xAE {
713 return true;
714 }
715 } else if second == 0x81 {
716 if third == 0xA7 {
717 return true;
718 }
719 }
720 }
721 0xEF => {
722 let new_read = read + 3;
724 if new_read > src.len() {
725 return true;
726 }
727 let second = unsafe { *(src.get_unchecked(read + 1)) };
728 let third = unsafe { *(src.get_unchecked(read + 2)) };
729 if ((UTF8_DATA.table[usize::from(second)]
730 & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
731 | (third >> 6))
732 != 2
733 {
734 return true;
735 }
736 if in_inclusive_range8(second, 0xAC, 0xB7) {
737 if second == 0xAC {
738 if third > 0x9C {
739 return true;
740 }
741 } else {
742 return true;
743 }
744 } else if in_inclusive_range8(second, 0xB9, 0xBB) {
745 if second == 0xB9 {
746 if third > 0xAF {
747 return true;
748 }
749 } else if second == 0xBB {
750 if third != 0xBF {
751 return true;
752 }
753 } else {
754 return true;
755 }
756 }
757 }
758 0xE0 => {
759 let new_read = read + 3;
761 if new_read > src.len() {
762 return true;
763 }
764 let second = unsafe { *(src.get_unchecked(read + 1)) };
765 let third = unsafe { *(src.get_unchecked(read + 2)) };
766 if ((UTF8_DATA.table[usize::from(second)]
767 & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
768 | (third >> 6))
769 != 2
770 {
771 return true;
772 }
773 if second < 0xA4 {
775 return true;
776 }
777 }
778 0xED => {
779 let new_read = read + 3;
781 if new_read > src.len() {
782 return true;
783 }
784 let second = unsafe { *(src.get_unchecked(read + 1)) };
785 let third = unsafe { *(src.get_unchecked(read + 2)) };
786 if ((UTF8_DATA.table[usize::from(second)]
787 & unsafe { *(UTF8_DATA.table.get_unchecked(byte as usize + 0x80)) })
788 | (third >> 6))
789 != 2
790 {
791 return true;
792 }
793 }
794 _ => {
795 return true;
797 }
798 }
799 return false;
800 } else {
801 return false;
802 }
803 }
804}
805
806#[allow(clippy::collapsible_if)]
819#[inline]
820#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
821pub fn is_str_bidi(buffer: &str) -> bool {
822 let mut bytes = buffer.as_bytes();
852 'outer: loop {
853 if let Some((mut byte, mut read)) = validate_ascii(bytes) {
860 'inner: loop {
861 if byte < 0xE0 {
863 if byte >= 0x80 {
864 if unlikely(byte >= 0xD6) {
868 if byte == 0xD6 {
869 let second = bytes[read + 1];
870 if second > 0x8F {
871 return true;
872 }
873 } else {
874 return true;
875 }
876 }
877 read += 2;
878 } else {
879 read += 1;
881 bytes = &bytes[read..];
887 continue 'outer;
888 }
889 } else if byte < 0xF0 {
890 if unlikely(!in_inclusive_range8(byte, 0xE3, 0xEE) && byte != 0xE1) {
892 let second = bytes[read + 1];
893 if byte == 0xE0 {
894 if second < 0xA4 {
895 return true;
896 }
897 } else if byte == 0xE2 {
898 let third = bytes[read + 2];
899 if second == 0x80 {
900 if third == 0x8F || third == 0xAB || third == 0xAE {
901 return true;
902 }
903 } else if second == 0x81 {
904 if third == 0xA7 {
905 return true;
906 }
907 }
908 } else {
909 debug_assert_eq!(byte, 0xEF);
910 if in_inclusive_range8(second, 0xAC, 0xB7) {
911 if second == 0xAC {
912 let third = bytes[read + 2];
913 if third > 0x9C {
914 return true;
915 }
916 } else {
917 return true;
918 }
919 } else if in_inclusive_range8(second, 0xB9, 0xBB) {
920 if second == 0xB9 {
921 let third = bytes[read + 2];
922 if third > 0xAF {
923 return true;
924 }
925 } else if second == 0xBB {
926 let third = bytes[read + 2];
927 if third != 0xBF {
928 return true;
929 }
930 } else {
931 return true;
932 }
933 }
934 }
935 }
936 read += 3;
937 } else {
938 let second = bytes[read + 1];
940 if unlikely(byte == 0xF0 && (second == 0x90 || second == 0x9E)) {
941 let third = bytes[read + 2];
942 if third >= 0xA0 {
943 return true;
944 }
945 }
946 read += 4;
947 }
948 if read >= bytes.len() {
952 return false;
953 }
954 byte = bytes[read];
955 continue 'inner;
956 }
957 } else {
958 return false;
959 }
960 }
961}
962
963pub fn is_utf16_bidi(buffer: &[u16]) -> bool {
981 is_utf16_bidi_impl(buffer)
982}
983
984#[inline(always)]
996pub fn is_char_bidi(c: char) -> bool {
997 let code_point = u32::from(c);
1017 if code_point < 0x0590 {
1018 return false;
1020 }
1021 if in_range32(code_point, 0x0900, 0xFB1D) {
1022 if in_inclusive_range32(code_point, 0x200F, 0x2067) {
1024 return code_point == 0x200F
1026 || code_point == 0x202B
1027 || code_point == 0x202E
1028 || code_point == 0x2067;
1029 }
1030 return false;
1031 }
1032 if code_point > 0x1EFFF {
1033 return false;
1035 }
1036 if in_range32(code_point, 0x11000, 0x1E800) {
1037 return false;
1039 }
1040 if in_range32(code_point, 0xFEFF, 0x10800) {
1041 return false;
1044 }
1045 if in_range32(code_point, 0xFE00, 0xFE70) {
1046 return false;
1048 }
1049 true
1050}
1051
1052#[inline(always)]
1071pub fn is_utf16_code_unit_bidi(u: u16) -> bool {
1072 if u < 0x0590 {
1073 return false;
1075 }
1076 if in_range16(u, 0x0900, 0xD802) {
1077 if in_inclusive_range16(u, 0x200F, 0x2067) {
1079 return u == 0x200F || u == 0x202B || u == 0x202E || u == 0x2067;
1081 }
1082 return false;
1083 }
1084 if in_range16(u, 0xD83C, 0xFB1D) {
1085 return false;
1088 }
1089 if in_range16(u, 0xD804, 0xD83A) {
1090 return false;
1092 }
1093 if u > 0xFEFE {
1094 return false;
1096 }
1097 if in_range16(u, 0xFE00, 0xFE70) {
1098 return false;
1100 }
1101 true
1102}
1103
1104pub fn check_utf8_for_latin1_and_bidi(buffer: &[u8]) -> Latin1Bidi {
1113 if let Some(offset) = is_utf8_latin1_impl(buffer) {
1114 if is_utf8_bidi(&buffer[offset..]) {
1115 Latin1Bidi::Bidi
1116 } else {
1117 Latin1Bidi::LeftToRight
1118 }
1119 } else {
1120 Latin1Bidi::Latin1
1121 }
1122}
1123
1124pub fn check_str_for_latin1_and_bidi(buffer: &str) -> Latin1Bidi {
1133 if let Some(offset) = is_str_latin1_impl(buffer) {
1136 if is_str_bidi(&buffer[offset..]) {
1137 Latin1Bidi::Bidi
1138 } else {
1139 Latin1Bidi::LeftToRight
1140 }
1141 } else {
1142 Latin1Bidi::Latin1
1143 }
1144}
1145
1146pub fn check_utf16_for_latin1_and_bidi(buffer: &[u16]) -> Latin1Bidi {
1155 check_utf16_for_latin1_and_bidi_impl(buffer)
1156}
1157
1158#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
1170pub fn convert_utf8_to_utf16(src: &[u8], dst: &mut [u16]) -> usize {
1171 assert!(dst.len() > src.len());
1174 let mut decoder = Utf8Decoder::new_inner();
1175 let mut total_read = 0usize;
1176 let mut total_written = 0usize;
1177 loop {
1178 let (result, read, written) =
1179 decoder.decode_to_utf16_raw(&src[total_read..], &mut dst[total_written..], true);
1180 total_read += read;
1181 total_written += written;
1182 match result {
1183 DecoderResult::InputEmpty => {
1184 return total_written;
1185 }
1186 DecoderResult::OutputFull => {
1187 unreachable!("The assert at the top of the function should have caught this.");
1188 }
1189 DecoderResult::Malformed(_, _) => {
1190 dst[total_written] = 0xFFFD;
1193 total_written += 1;
1194 }
1195 }
1196 }
1197}
1198
1199#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
1210pub fn convert_str_to_utf16(src: &str, dst: &mut [u16]) -> usize {
1211 assert!(
1212 dst.len() >= src.len(),
1213 "Destination must not be shorter than the source."
1214 );
1215 let bytes = src.as_bytes();
1216 let mut read = 0;
1217 let mut written = 0;
1218 'outer: loop {
1219 let mut byte = {
1220 let src_remaining = &bytes[read..];
1221 let dst_remaining = &mut dst[written..];
1222 let length = src_remaining.len();
1223 match ascii_to_basic_latin(src_remaining, dst_remaining) {
1224 None => {
1225 written += length;
1226 return written;
1227 }
1228 Some((non_ascii, consumed)) => {
1229 read += consumed;
1230 written += consumed;
1231 non_ascii
1232 }
1233 }
1234 };
1235 'inner: loop {
1236 if byte < 0xE0 {
1238 if byte >= 0x80 {
1239 let second = unsafe { *(bytes.get_unchecked(read + 1)) };
1241 let point = ((u16::from(byte) & 0x1F) << 6) | (u16::from(second) & 0x3F);
1242 unsafe { *(dst.get_unchecked_mut(written)) = point };
1243 read += 2;
1244 written += 1;
1245 } else {
1246 unsafe { *(dst.get_unchecked_mut(written)) = u16::from(byte) };
1248 read += 1;
1249 written += 1;
1250 continue 'outer;
1256 }
1257 } else if byte < 0xF0 {
1258 let second = unsafe { *(bytes.get_unchecked(read + 1)) };
1260 let third = unsafe { *(bytes.get_unchecked(read + 2)) };
1261 let point = ((u16::from(byte) & 0xF) << 12)
1262 | ((u16::from(second) & 0x3F) << 6)
1263 | (u16::from(third) & 0x3F);
1264 unsafe { *(dst.get_unchecked_mut(written)) = point };
1265 read += 3;
1266 written += 1;
1267 } else {
1268 let second = unsafe { *(bytes.get_unchecked(read + 1)) };
1270 let third = unsafe { *(bytes.get_unchecked(read + 2)) };
1271 let fourth = unsafe { *(bytes.get_unchecked(read + 3)) };
1272 let point = ((u32::from(byte) & 0x7) << 18)
1273 | ((u32::from(second) & 0x3F) << 12)
1274 | ((u32::from(third) & 0x3F) << 6)
1275 | (u32::from(fourth) & 0x3F);
1276 unsafe { *(dst.get_unchecked_mut(written)) = (0xD7C0 + (point >> 10)) as u16 };
1277 unsafe {
1278 *(dst.get_unchecked_mut(written + 1)) = (0xDC00 + (point & 0x3FF)) as u16
1279 };
1280 read += 4;
1281 written += 2;
1282 }
1283 if read >= src.len() {
1287 return written;
1288 }
1289 byte = bytes[read];
1290 continue 'inner;
1291 }
1292 }
1293}
1294
1295pub fn convert_utf8_to_utf16_without_replacement(src: &[u8], dst: &mut [u16]) -> Option<usize> {
1308 assert!(
1309 dst.len() >= src.len(),
1310 "Destination must not be shorter than the source."
1311 );
1312 let (read, written) = convert_utf8_to_utf16_up_to_invalid(src, dst);
1313 if read == src.len() {
1314 return Some(written);
1315 }
1316 None
1317}
1318
1319#[inline(always)]
1345pub fn convert_utf16_to_utf8_partial(src: &[u16], dst: &mut [u8]) -> (usize, usize) {
1346 let (read, written) = convert_utf16_to_utf8_partial_inner(src, dst);
1356 if likely(read == src.len()) {
1357 return (read, written);
1358 }
1359 let (tail_read, tail_written) =
1360 convert_utf16_to_utf8_partial_tail(&src[read..], &mut dst[written..]);
1361 (read + tail_read, written + tail_written)
1362}
1363
1364#[inline(always)]
1382pub fn convert_utf16_to_utf8(src: &[u16], dst: &mut [u8]) -> usize {
1383 assert!(dst.len() >= src.len() * 3);
1384 let (read, written) = convert_utf16_to_utf8_partial(src, dst);
1385 debug_assert_eq!(read, src.len());
1386 written
1387}
1388
1389pub fn convert_utf16_to_str_partial(src: &[u16], dst: &mut str) -> (usize, usize) {
1403 let mut bytes = scopeguard::guard(unsafe { dst.as_bytes_mut() }, |bytes| {
1415 bytes.iter_mut().for_each(|b| *b = 0)
1416 });
1417 let (read, written) = convert_utf16_to_utf8_partial(src, &mut bytes);
1418 let len = bytes.len();
1419 let mut trail = written;
1420 while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
1421 bytes[trail] = 0;
1422 trail += 1;
1423 }
1424 let _ = scopeguard::ScopeGuard::<&mut [u8], _>::into_inner(bytes);
1426 (read, written)
1427}
1428
1429#[inline(always)]
1442pub fn convert_utf16_to_str(src: &[u16], dst: &mut str) -> usize {
1443 assert!(dst.len() >= src.len() * 3);
1444 let (read, written) = convert_utf16_to_str_partial(src, dst);
1445 debug_assert_eq!(read, src.len());
1446 written
1447}
1448
1449#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
1461pub fn convert_latin1_to_utf16(src: &[u8], dst: &mut [u16]) {
1462 assert!(
1463 dst.len() >= src.len(),
1464 "Destination must not be shorter than the source."
1465 );
1466 unpack_latin1(src, dst);
1467}
1468
1469pub fn convert_latin1_to_utf8_partial(src: &[u8], dst: &mut [u8]) -> (usize, usize) {
1483 let src_len = src.len();
1484 let dst_len = dst.len();
1485 let dst_len_capped = dst_len.saturating_sub(1);
1488 let mut total_read = 0usize;
1489 let mut total_written = 0usize;
1490 loop {
1491 if total_written < dst_len_capped {
1492 let src_left = src_len - total_read;
1494 let dst_left = dst_len_capped - total_written;
1495 let min_left = ::core::cmp::min(src_left, dst_left);
1496 if let Some((non_ascii, consumed)) =
1497 { ascii_to_ascii(&src[total_read..], &mut dst[total_written..dst_len_capped]) }
1498 {
1499 total_read += consumed;
1500 total_written += consumed;
1501 if total_written.checked_add(2).unwrap() > dst_len {
1502 return (total_read, total_written);
1503 }
1504
1505 total_read += 1; dst[total_written] = (non_ascii >> 6) | 0xC0;
1508 total_written += 1;
1509 dst[total_written] = (non_ascii & 0x3F) | 0x80;
1510 total_written += 1;
1511 continue;
1512 }
1513 total_read += min_left;
1514 total_written += min_left;
1515 }
1516 if total_read < src.len() && total_written < dst.len() {
1517 let b = src[total_read];
1518 if b.is_ascii() {
1519 dst[total_written] = b;
1520 total_read += 1;
1521 total_written += 1;
1522 }
1523 }
1524 return (total_read, total_written);
1525 }
1526}
1527
1528#[inline]
1547pub fn convert_latin1_to_utf8(src: &[u8], dst: &mut [u8]) -> usize {
1548 assert!(
1549 dst.len() >= src.len() * 2,
1550 "Destination must not be shorter than the source times two."
1551 );
1552 let (read, written) = convert_latin1_to_utf8_partial(src, dst);
1553 debug_assert_eq!(read, src.len());
1554 written
1555}
1556
1557#[inline]
1566pub fn convert_latin1_to_str_partial(src: &[u8], dst: &mut str) -> (usize, usize) {
1567 let mut bytes = scopeguard::guard(unsafe { dst.as_bytes_mut() }, |bytes| {
1573 bytes.iter_mut().for_each(|b| *b = 0)
1574 });
1575 let (read, written) = convert_latin1_to_utf8_partial(src, &mut bytes);
1576 let len = bytes.len();
1577 let mut trail = written;
1578 let max = ::core::cmp::min(len, trail + MAX_STRIDE_SIZE);
1579 while trail < max {
1580 bytes[trail] = 0;
1581 trail += 1;
1582 }
1583 while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
1584 bytes[trail] = 0;
1585 trail += 1;
1586 }
1587 let _ = scopeguard::ScopeGuard::<&mut [u8], _>::into_inner(bytes);
1589 (read, written)
1590}
1591
1592#[inline]
1605pub fn convert_latin1_to_str(src: &[u8], dst: &mut str) -> usize {
1606 assert!(
1607 dst.len() >= src.len() * 2,
1608 "Destination must not be shorter than the source times two."
1609 );
1610 let (read, written) = convert_latin1_to_str_partial(src, dst);
1611 debug_assert_eq!(read, src.len());
1612 written
1613}
1614
1615pub fn convert_utf8_to_latin1_lossy(src: &[u8], dst: &mut [u8]) -> usize {
1639 assert!(
1640 dst.len() >= src.len(),
1641 "Destination must not be shorter than the source."
1642 );
1643 non_fuzz_debug_assert!(is_utf8_latin1(src));
1644 let src_len = src.len();
1645 let mut total_read = 0usize;
1646 let mut total_written = 0usize;
1647 loop {
1648 let src_left = src_len - total_read;
1650 if let Some((non_ascii, consumed)) =
1651 { ascii_to_ascii(&src[total_read..], &mut dst[total_written..]) }
1652 {
1653 total_read += consumed + 1;
1654 total_written += consumed;
1655
1656 if total_read == src_len {
1657 return total_written;
1658 }
1659
1660 let trail = src[total_read];
1661 total_read += 1;
1662
1663 dst[total_written] = ((non_ascii & 0x1F) << 6) | (trail & 0x3F);
1664 total_written += 1;
1665 continue;
1666 }
1667 return total_written + src_left;
1668 }
1669}
1670
1671pub fn convert_utf16_to_latin1_lossy(src: &[u16], dst: &mut [u8]) {
1695 assert!(
1696 dst.len() >= src.len(),
1697 "Destination must not be shorter than the source."
1698 );
1699 pack_latin1(src, dst);
1701}
1702
1703#[cfg(feature = "alloc")]
1711pub fn decode_latin1<'a>(bytes: &'a [u8]) -> Cow<'a, str> {
1712 let up_to = ascii_valid_up_to(bytes);
1713 if up_to >= bytes.len() {
1715 debug_assert_eq!(up_to, bytes.len());
1716 let s: &str = unsafe { ::core::str::from_utf8_unchecked(bytes) };
1719 return Cow::Borrowed(s);
1720 }
1721 let (head, tail) = bytes.split_at(up_to);
1722 let capacity = head.len() + tail.len() * 2;
1723 let mut vec = Vec::with_capacity(capacity);
1724 vec.extend_from_slice(head);
1725 let old_len = vec.len();
1726 let spare_capacity = crate::minimally_init(vec.spare_capacity_mut());
1727 debug_assert_eq!(old_len, up_to);
1728 let written = convert_latin1_to_utf8(tail, spare_capacity);
1729 debug_assert!(written <= spare_capacity.len());
1730 let new_len = old_len + written;
1731 assert!(new_len <= vec.capacity());
1732 unsafe {
1741 vec.set_len(new_len);
1742 }
1743 Cow::Owned(unsafe { String::from_utf8_unchecked(vec) })
1747}
1748
1749#[cfg(feature = "alloc")]
1766pub fn encode_latin1_lossy<'a>(string: &'a str) -> Cow<'a, [u8]> {
1767 let bytes = string.as_bytes();
1768 let up_to = ascii_valid_up_to(bytes);
1769 if up_to >= bytes.len() {
1771 debug_assert_eq!(up_to, bytes.len());
1772 return Cow::Borrowed(bytes);
1773 }
1774 let (head, tail) = bytes.split_at(up_to);
1775 let capacity = bytes.len();
1776 let mut vec = Vec::with_capacity(capacity);
1777 vec.extend_from_slice(head);
1778 let old_len = vec.len();
1779 let spare_capacity = crate::minimally_init(vec.spare_capacity_mut());
1780 debug_assert_eq!(old_len, up_to);
1781 let written = convert_utf8_to_latin1_lossy(tail, spare_capacity);
1782 debug_assert!(written <= spare_capacity.len());
1783 let new_len = old_len + written;
1784 assert!(new_len <= vec.capacity());
1785 unsafe {
1792 vec.set_len(new_len);
1793 }
1794 Cow::Owned(vec)
1795}
1796
1797cfg_if! {
1798 if #[cfg(all(
1799 feature = "simd-accel",
1800 target_endian = "little",
1801 ))] {
1802 pub(crate) use crate::simd_funcs::validate_bmp_stride;
1803 pub(crate) use crate::simd_funcs::validate_latin1_str_stride;
1804
1805 #[inline(always)]
1806 #[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
1807 fn is_str_latin1_impl(buffer: &str) -> Option<usize> {
1808 let mut consumed = 0;
1809 let (strides, tail) = buffer.as_bytes().as_chunks::<STRIDE>();
1810 for stride in strides.iter() {
1811 if let Some(pos) = validate_latin1_str_stride(stride) {
1812 return Some(consumed + pos);
1813 }
1814 consumed += STRIDE;
1815 }
1816 for slot in tail.iter() {
1817 if *slot > 0xC3 {
1818 return Some(consumed);
1819 }
1820 consumed += 1;
1821 }
1822 None
1823 }
1824 } else {
1825 #[inline(always)]
1826 pub(crate) fn validate_bmp_stride(stride: &[u16; STRIDE]) -> Option<usize> {
1827 if (stride[0] & 0xF800 != 0xD800)
1828 && (stride[1] & 0xF800 != 0xD800)
1829 && (stride[2] & 0xF800 != 0xD800)
1830 && (stride[3] & 0xF800 != 0xD800)
1831 && (stride[4] & 0xF800 != 0xD800)
1832 && (stride[5] & 0xF800 != 0xD800)
1833 && (stride[6] & 0xF800 != 0xD800)
1834 && (stride[7] & 0xF800 != 0xD800)
1835 && (stride[8] & 0xF800 != 0xD800)
1836 && (stride[9] & 0xF800 != 0xD800)
1837 && (stride[10] & 0xF800 != 0xD800)
1838 && (stride[11] & 0xF800 != 0xD800)
1839 && (stride[12] & 0xF800 != 0xD800)
1840 && (stride[13] & 0xF800 != 0xD800)
1841 && (stride[14] & 0xF800 != 0xD800)
1842 && (stride[15] & 0xF800 != 0xD800)
1843 {
1844 return None;
1845 }
1846 for (i, c) in stride.iter().enumerate() {
1847 if c & 0xF800 == 0xD800 {
1848 return Some(i);
1849 }
1850 }
1851 debug_assert!(false);
1852 None
1853 }
1854
1855 #[inline(always)]
1856 fn is_str_latin1_impl(buffer: &str) -> Option<usize> {
1857 let mut bytes = buffer.as_bytes();
1858 let mut total = 0;
1859 loop {
1860 if let Some((byte, offset)) = validate_ascii(bytes) {
1861 total += offset;
1862 if byte > 0xC3 {
1863 return Some(total);
1864 }
1865 bytes = &bytes[offset + 2..];
1866 total += 2;
1867 } else {
1868 return None;
1869 }
1870 }
1871 }
1872 }
1873}
1874
1875#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
1878pub fn utf16_valid_up_to(buffer: &[u16]) -> usize {
1879 let mut consumed = 0usize;
1880 'outer: loop {
1881 let (strides, tail) = &buffer[consumed..].as_chunks::<STRIDE>();
1882 #[allow(clippy::never_loop)]
1884 'bmp: loop {
1885 for stride in strides.iter() {
1886 if let Some(pos) = validate_bmp_stride(stride) {
1887 consumed += pos;
1888 break 'bmp;
1889 }
1890 consumed += STRIDE;
1891 }
1892 for slot in tail.iter() {
1893 if *slot & 0xF800 == 0xD800 {
1894 break 'bmp;
1895 }
1896 consumed += 1;
1897 }
1898 debug_assert_eq!(consumed, buffer.len());
1899 return consumed;
1900 }
1901 let mut unit = buffer[consumed];
1903 let mut unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
1904 debug_assert!(unit_minus_surrogate_start <= (0xDFFF - 0xD800));
1905 'surrogate: loop {
1906 if unit_minus_surrogate_start > (0xDBFF - 0xD800) {
1907 return consumed;
1909 }
1910 let next = consumed + 1;
1912 if next == buffer.len() {
1913 return consumed;
1915 }
1916 let second = buffer[next];
1917 let second_minus_low_surrogate_start = second.wrapping_sub(0xDC00);
1918 if second_minus_low_surrogate_start > (0xDFFF - 0xDC00) {
1919 return consumed;
1922 }
1923 consumed = next + 1;
1925 loop {
1929 if consumed == buffer.len() {
1930 return consumed;
1932 }
1933 unit = buffer[consumed];
1934 unit_minus_surrogate_start = unit.wrapping_sub(0xD800);
1935 if unit_minus_surrogate_start <= (0xDFFF - 0xD800) {
1936 continue 'surrogate;
1937 }
1938 consumed += 1;
1940 if unit == 0x0020 {
1942 continue;
1943 }
1944 continue 'outer;
1945 }
1946 }
1947 }
1948}
1949
1950pub fn utf8_latin1_up_to(buffer: &[u8]) -> usize {
1954 is_utf8_latin1_impl(buffer).unwrap_or(buffer.len())
1955}
1956
1957pub fn str_latin1_up_to(buffer: &str) -> usize {
1960 is_str_latin1_impl(buffer).unwrap_or(buffer.len())
1961}
1962
1963#[inline]
1965pub fn ensure_utf16_validity(buffer: &mut [u16]) {
1966 let mut offset = 0;
1967 loop {
1968 offset += utf16_valid_up_to(&buffer[offset..]);
1969 if offset == buffer.len() {
1970 return;
1971 }
1972 buffer[offset] = 0xFFFD;
1973 offset += 1;
1974 }
1975}
1976
1977pub fn copy_ascii_to_ascii(src: &[u8], dst: &mut [u8]) -> usize {
1989 assert!(
1990 dst.len() >= src.len(),
1991 "Destination must not be shorter than the source."
1992 );
1993 if let Some((_, consumed)) = { ascii_to_ascii(src, dst) } {
1994 consumed
1995 } else {
1996 src.len()
1997 }
1998}
1999
2000#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
2013pub fn copy_ascii_to_basic_latin(src: &[u8], dst: &mut [u16]) -> usize {
2014 assert!(
2015 dst.len() >= src.len(),
2016 "Destination must not be shorter than the source."
2017 );
2018 if let Some((_, consumed)) = { ascii_to_basic_latin(src, dst) } {
2019 consumed
2020 } else {
2021 src.len()
2022 }
2023}
2024
2025#[crate::multiversion(targets("x86_64+avx2+bmi1", "x86+avx2+bmi1"))]
2038pub fn copy_basic_latin_to_ascii(src: &[u16], dst: &mut [u8]) -> usize {
2039 assert!(
2040 dst.len() >= src.len(),
2041 "Destination must not be shorter than the source."
2042 );
2043 if let Some((_, consumed)) = { basic_latin_to_ascii(src, dst) } {
2044 consumed
2045 } else {
2046 src.len()
2047 }
2048}
2049
2050#[cfg(all(test, feature = "alloc"))]
2054mod tests {
2055 use super::*;
2056
2057 #[test]
2058 fn test_is_ascii_success() {
2059 let mut src: Vec<u8> = Vec::with_capacity(128);
2060 src.resize(128, 0);
2061 for i in 0..src.len() {
2062 src[i] = i as u8;
2063 }
2064 for i in 0..src.len() {
2065 assert!(is_ascii(&src[i..]));
2066 }
2067 }
2068
2069 #[test]
2070 fn test_is_ascii_fail() {
2071 let mut src: Vec<u8> = Vec::with_capacity(128);
2072 src.resize(128, 0);
2073 for i in 0..src.len() {
2074 src[i] = i as u8;
2075 }
2076 for i in 0..src.len() {
2077 let tail = &mut src[i..];
2078 for j in 0..tail.len() {
2079 tail[j] = 0xA0;
2080 assert!(!is_ascii(tail));
2081 }
2082 }
2083 }
2084
2085 #[test]
2086 fn test_is_basic_latin_success() {
2087 let mut src: Vec<u16> = Vec::with_capacity(128);
2088 src.resize(128, 0);
2089 for i in 0..src.len() {
2090 src[i] = i as u16;
2091 }
2092 for i in 0..src.len() {
2093 assert!(is_basic_latin(&src[i..]));
2094 }
2095 }
2096
2097 #[test]
2098 fn test_is_basic_latin_fail() {
2099 let mut src: Vec<u16> = Vec::with_capacity(128);
2100 src.resize(128, 0);
2101 for i in 0..src.len() {
2102 src[i] = i as u16;
2103 }
2104 for i in 0..src.len() {
2105 let tail = &mut src[i..];
2106 for j in 0..tail.len() {
2107 tail[j] = 0xA0;
2108 assert!(!is_basic_latin(tail));
2109 }
2110 }
2111 }
2112
2113 #[test]
2114 fn test_is_utf16_latin1_success() {
2115 let mut src: Vec<u16> = Vec::with_capacity(256);
2116 src.resize(256, 0);
2117 for i in 0..src.len() {
2118 src[i] = i as u16;
2119 }
2120 for i in 0..src.len() {
2121 assert!(is_utf16_latin1(&src[i..]));
2122 assert_eq!(
2123 check_utf16_for_latin1_and_bidi(&src[i..]),
2124 Latin1Bidi::Latin1
2125 );
2126 }
2127 }
2128
2129 #[test]
2130 fn test_is_utf16_latin1_fail() {
2131 let len = if cfg!(miri) { 64 } else { 256 }; let mut src: Vec<u16> = Vec::with_capacity(len);
2133 src.resize(len, 0);
2134 for i in 0..src.len() {
2135 src[i] = i as u16;
2136 }
2137 for i in 0..src.len() {
2138 let tail = &mut src[i..];
2139 for j in 0..tail.len() {
2140 tail[j] = 0x100 + j as u16;
2141 assert!(!is_utf16_latin1(tail));
2142 assert_ne!(check_utf16_for_latin1_and_bidi(tail), Latin1Bidi::Latin1);
2143 }
2144 }
2145 }
2146
2147 #[test]
2148 fn test_is_str_latin1_success() {
2149 let len = if cfg!(miri) { 64 } else { 256 }; let mut src: Vec<u16> = Vec::with_capacity(len);
2151 src.resize(len, 0);
2152 for i in 0..src.len() {
2153 src[i] = i as u16;
2154 }
2155 for i in 0..src.len() {
2156 let s = String::from_utf16(&src[i..]).unwrap();
2157 assert!(is_str_latin1(&s[..]));
2158 assert_eq!(check_str_for_latin1_and_bidi(&s[..]), Latin1Bidi::Latin1);
2159 }
2160 }
2161
2162 #[test]
2163 fn test_is_str_latin1_fail() {
2164 let len = if cfg!(miri) { 32 } else { 256 }; let mut src: Vec<u16> = Vec::with_capacity(len);
2166 src.resize(len, 0);
2167 for i in 0..src.len() {
2168 src[i] = i as u16;
2169 }
2170 for i in 0..src.len() {
2171 let tail = &mut src[i..];
2172 for j in 0..tail.len() {
2173 tail[j] = 0x100 + j as u16;
2174 let s = String::from_utf16(tail).unwrap();
2175 assert!(!is_str_latin1(&s[..]));
2176 assert_ne!(check_str_for_latin1_and_bidi(&s[..]), Latin1Bidi::Latin1);
2177 }
2178 }
2179 }
2180
2181 #[test]
2182 fn test_is_utf8_latin1_success() {
2183 let len = if cfg!(miri) { 64 } else { 256 }; let mut src: Vec<u16> = Vec::with_capacity(len);
2185 src.resize(len, 0);
2186 for i in 0..src.len() {
2187 src[i] = i as u16;
2188 }
2189 for i in 0..src.len() {
2190 let s = String::from_utf16(&src[i..]).unwrap();
2191 assert!(is_utf8_latin1(s.as_bytes()));
2192 assert_eq!(
2193 check_utf8_for_latin1_and_bidi(s.as_bytes()),
2194 Latin1Bidi::Latin1
2195 );
2196 }
2197 }
2198
2199 #[test]
2200 fn test_is_utf8_latin1_fail() {
2201 let len = if cfg!(miri) { 32 } else { 256 }; let mut src: Vec<u16> = Vec::with_capacity(len);
2203 src.resize(len, 0);
2204 for i in 0..src.len() {
2205 src[i] = i as u16;
2206 }
2207 for i in 0..src.len() {
2208 let tail = &mut src[i..];
2209 for j in 0..tail.len() {
2210 tail[j] = 0x100 + j as u16;
2211 let s = String::from_utf16(tail).unwrap();
2212 assert!(!is_utf8_latin1(s.as_bytes()));
2213 assert_ne!(
2214 check_utf8_for_latin1_and_bidi(s.as_bytes()),
2215 Latin1Bidi::Latin1
2216 );
2217 }
2218 }
2219 }
2220
2221 #[test]
2222 fn test_is_utf8_latin1_invalid() {
2223 assert!(!is_utf8_latin1(b"\xC3"));
2224 assert!(!is_utf8_latin1(b"a\xC3"));
2225 assert!(!is_utf8_latin1(b"\xFF"));
2226 assert!(!is_utf8_latin1(b"a\xFF"));
2227 assert!(!is_utf8_latin1(b"\xC3\xFF"));
2228 assert!(!is_utf8_latin1(b"a\xC3\xFF"));
2229 }
2230
2231 #[test]
2232 fn test_convert_utf8_to_utf16() {
2233 let src = "abcdefghijklmnopqrstu\u{1F4A9}v\u{2603}w\u{00B6}xyzz";
2234 let mut dst: Vec<u16> = Vec::with_capacity(src.len() + 1);
2235 dst.resize(src.len() + 1, 0);
2236 let len = convert_utf8_to_utf16(src.as_bytes(), &mut dst[..]);
2237 dst.truncate(len);
2238 let reference: Vec<u16> = src.encode_utf16().collect();
2239 assert_eq!(dst, reference);
2240 }
2241
2242 #[test]
2243 fn test_convert_str_to_utf16() {
2244 let src = "abcdefghijklmnopqrstu\u{1F4A9}v\u{2603}w\u{00B6}xyzz";
2245 let mut dst: Vec<u16> = Vec::with_capacity(src.len());
2246 dst.resize(src.len(), 0);
2247 let len = convert_str_to_utf16(src, &mut dst[..]);
2248 dst.truncate(len);
2249 let reference: Vec<u16> = src.encode_utf16().collect();
2250 assert_eq!(dst, reference);
2251 }
2252
2253 #[test]
2254 fn test_convert_utf16_to_utf8_partial() {
2255 let reference = "abcdefghijklmnopqrstu\u{1F4A9}v\u{2603}w\u{00B6}xyzz";
2256 let src: Vec<u16> = reference.encode_utf16().collect();
2257 let mut dst: Vec<u8> = Vec::with_capacity(src.len() * 3 + 1);
2258 dst.resize(src.len() * 3 + 1, 0);
2259 let (read, written) = convert_utf16_to_utf8_partial(&src[..], &mut dst[..24]);
2260 let len = written + convert_utf16_to_utf8(&src[read..], &mut dst[written..]);
2261 dst.truncate(len);
2262 assert_eq!(dst, reference.as_bytes());
2263 }
2264
2265 #[test]
2266 fn test_convert_utf16_to_utf8_partial_leaves_tail_unmodified() {
2267 for &non_ascii in &[0x00CAu16, 0x00E4, 0x4E00, 0xD83D, 0xDCA9] {
2271 for prefix_len in 0..70 {
2272 let mut src = vec![u16::from(b'a'); prefix_len];
2273 src.push(non_ascii);
2274 src.extend_from_slice(&[u16::from(b'b'); 40]);
2275 for dst_len in 0..=(src.len() * 3) {
2276 let mut dst = vec![b'q'; dst_len];
2277 let (_, written) = convert_utf16_to_utf8_partial(&src[..], &mut dst[..]);
2278 assert!(dst[written..].iter().all(|&b| b == b'q'));
2279
2280 let mut string = "q".repeat(dst_len);
2281 convert_utf16_to_str_partial(&src[..], &mut string[..]);
2282 assert!(core::str::from_utf8(string.as_bytes()).is_ok());
2283 }
2284 }
2285 }
2286 }
2287
2288 #[test]
2289 fn test_convert_latin1_to_utf8_partial_leaves_tail_unmodified() {
2290 for &non_ascii in &[0x80u8, 0xA0, 0xCA, 0xE4, 0xFF] {
2294 for prefix_len in 0..70 {
2295 let mut src = vec![b'a'; prefix_len];
2296 src.push(non_ascii);
2297 src.extend_from_slice(&[b'b'; 40]);
2298 for dst_len in 0..=(src.len() * 2) {
2299 let mut dst = vec![b'q'; dst_len];
2300 let (_, written) = convert_latin1_to_utf8_partial(&src[..], &mut dst[..]);
2301 assert!(dst[written..].iter().all(|&b| b == b'q'));
2302
2303 let mut string = "q".repeat(dst_len);
2304 convert_latin1_to_str_partial(&src[..], &mut string[..]);
2305 assert!(core::str::from_utf8(string.as_bytes()).is_ok());
2306 }
2307 }
2308 }
2309 }
2310
2311 #[test]
2312 fn test_convert_utf16_to_utf8() {
2313 let reference = "abcdefghijklmnopqrstu\u{1F4A9}v\u{2603}w\u{00B6}xyzz";
2314 let src: Vec<u16> = reference.encode_utf16().collect();
2315 let mut dst: Vec<u8> = Vec::with_capacity(src.len() * 3 + 1);
2316 dst.resize(src.len() * 3 + 1, 0);
2317 let len = convert_utf16_to_utf8(&src[..], &mut dst[..]);
2318 dst.truncate(len);
2319 assert_eq!(dst, reference.as_bytes());
2320 }
2321
2322 #[test]
2323 fn test_convert_latin1_to_utf16() {
2324 let mut src: Vec<u8> = Vec::with_capacity(256);
2325 src.resize(256, 0);
2326 let mut reference: Vec<u16> = Vec::with_capacity(256);
2327 reference.resize(256, 0);
2328 for i in 0..256 {
2329 src[i] = i as u8;
2330 reference[i] = i as u16;
2331 }
2332 let mut dst: Vec<u16> = Vec::with_capacity(src.len());
2333 dst.resize(src.len(), 0);
2334 convert_latin1_to_utf16(&src[..], &mut dst[..]);
2335 assert_eq!(dst, reference);
2336 }
2337
2338 #[test]
2339 fn test_convert_latin1_to_utf8_partial() {
2340 let mut dst = [0u8, 2];
2341 let (read, written) = convert_latin1_to_utf8_partial(b"a\xFF", &mut dst[..]);
2342 assert_eq!(read, 1);
2343 assert_eq!(written, 1);
2344 }
2345
2346 #[test]
2347 fn test_convert_latin1_to_utf8() {
2348 let mut src: Vec<u8> = Vec::with_capacity(256);
2349 src.resize(256, 0);
2350 let mut reference: Vec<u16> = Vec::with_capacity(256);
2351 reference.resize(256, 0);
2352 for i in 0..256 {
2353 src[i] = i as u8;
2354 reference[i] = i as u16;
2355 }
2356 let s = String::from_utf16(&reference[..]).unwrap();
2357 let mut dst: Vec<u8> = Vec::with_capacity(src.len() * 2);
2358 dst.resize(src.len() * 2, 0);
2359 let len = convert_latin1_to_utf8(&src[..], &mut dst[..]);
2360 dst.truncate(len);
2361 assert_eq!(&dst[..], s.as_bytes());
2362 }
2363
2364 #[test]
2365 fn test_convert_utf8_to_latin1_lossy() {
2366 let mut reference: Vec<u8> = Vec::with_capacity(256);
2367 reference.resize(256, 0);
2368 let mut src16: Vec<u16> = Vec::with_capacity(256);
2369 src16.resize(256, 0);
2370 for i in 0..256 {
2371 src16[i] = i as u16;
2372 reference[i] = i as u8;
2373 }
2374 let src = String::from_utf16(&src16[..]).unwrap();
2375 let mut dst: Vec<u8> = Vec::with_capacity(src.len());
2376 dst.resize(src.len(), 0);
2377 let len = convert_utf8_to_latin1_lossy(src.as_bytes(), &mut dst[..]);
2378 dst.truncate(len);
2379 assert_eq!(dst, reference);
2380 }
2381
2382 #[cfg(all(debug_assertions, not(fuzzing)))]
2383 #[test]
2384 #[should_panic]
2385 fn test_convert_utf8_to_latin1_lossy_panics() {
2386 let mut dst = [0u8; 16];
2387 let _ = convert_utf8_to_latin1_lossy("\u{100}".as_bytes(), &mut dst[..]);
2388 }
2389
2390 #[test]
2391 fn test_convert_utf16_to_latin1_lossy() {
2392 let mut src: Vec<u16> = Vec::with_capacity(256);
2393 src.resize(256, 0);
2394 let mut reference: Vec<u8> = Vec::with_capacity(256);
2395 reference.resize(256, 0);
2396 for i in 0..256 {
2397 src[i] = i as u16;
2398 reference[i] = i as u8;
2399 }
2400 let mut dst: Vec<u8> = Vec::with_capacity(src.len());
2401 dst.resize(src.len(), 0);
2402 convert_utf16_to_latin1_lossy(&src[..], &mut dst[..]);
2403 assert_eq!(dst, reference);
2404 }
2405
2406 #[test]
2407 fn test_convert_utf16_to_latin1_lossy_panics() {
2409 let mut dst = [0u8; 16];
2410 let _ = convert_utf16_to_latin1_lossy(&[0x0100u16], &mut dst[..]);
2411 }
2412
2413 #[test]
2414 fn test_utf16_valid_up_to() {
2415 let valid = vec![
2416 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0x2603u16,
2417 0xD83Du16, 0xDCA9u16, 0x00B6u16,
2418 ];
2419 assert_eq!(utf16_valid_up_to(&valid[..]), 16);
2420 let lone_high = vec![
2421 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2422 0x2603u16, 0xD83Du16, 0x00B6u16,
2423 ];
2424 assert_eq!(utf16_valid_up_to(&lone_high[..]), 14);
2425 let lone_low = vec![
2426 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2427 0x2603u16, 0xDCA9u16, 0x00B6u16,
2428 ];
2429 assert_eq!(utf16_valid_up_to(&lone_low[..]), 14);
2430 let lone_high_at_end = vec![
2431 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2432 0x2603u16, 0x00B6u16, 0xD83Du16,
2433 ];
2434 assert_eq!(utf16_valid_up_to(&lone_high_at_end[..]), 15);
2435 }
2436
2437 #[test]
2438 fn test_ensure_utf16_validity() {
2439 let mut src = vec![
2440 0u16, 0xD83Du16, 0u16, 0u16, 0u16, 0xD83Du16, 0xDCA9u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2441 0u16, 0xDCA9u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2442 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2443 ];
2444 let reference = vec![
2445 0u16, 0xFFFDu16, 0u16, 0u16, 0u16, 0xD83Du16, 0xDCA9u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2446 0u16, 0xFFFDu16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2447 0u16, 0u16, 0u16, 0u16, 0u16, 0u16,
2448 ];
2449 ensure_utf16_validity(&mut src[..]);
2450 assert_eq!(src, reference);
2451 }
2452
2453 #[test]
2454 fn test_is_char_bidi() {
2455 assert!(!is_char_bidi('a'));
2456 assert!(!is_char_bidi('\u{03B1}'));
2457 assert!(!is_char_bidi('\u{3041}'));
2458 assert!(!is_char_bidi('\u{1F4A9}'));
2459 assert!(!is_char_bidi('\u{FE00}'));
2460 assert!(!is_char_bidi('\u{202C}'));
2461 assert!(!is_char_bidi('\u{FEFF}'));
2462 assert!(is_char_bidi('\u{0590}'));
2463 assert!(is_char_bidi('\u{08FF}'));
2464 assert!(is_char_bidi('\u{061C}'));
2465 assert!(is_char_bidi('\u{FB50}'));
2466 assert!(is_char_bidi('\u{FDFF}'));
2467 assert!(is_char_bidi('\u{FE70}'));
2468 assert!(is_char_bidi('\u{FEFE}'));
2469 assert!(is_char_bidi('\u{200F}'));
2470 assert!(is_char_bidi('\u{202B}'));
2471 assert!(is_char_bidi('\u{202E}'));
2472 assert!(is_char_bidi('\u{2067}'));
2473 assert!(is_char_bidi('\u{10800}'));
2474 assert!(is_char_bidi('\u{10FFF}'));
2475 assert!(is_char_bidi('\u{1E800}'));
2476 assert!(is_char_bidi('\u{1EFFF}'));
2477 }
2478
2479 #[test]
2480 fn test_is_utf16_code_unit_bidi() {
2481 assert!(!is_utf16_code_unit_bidi(0x0062));
2482 assert!(!is_utf16_code_unit_bidi(0x03B1));
2483 assert!(!is_utf16_code_unit_bidi(0x3041));
2484 assert!(!is_utf16_code_unit_bidi(0xD801));
2485 assert!(!is_utf16_code_unit_bidi(0xFE00));
2486 assert!(!is_utf16_code_unit_bidi(0x202C));
2487 assert!(!is_utf16_code_unit_bidi(0xFEFF));
2488 assert!(is_utf16_code_unit_bidi(0x0590));
2489 assert!(is_utf16_code_unit_bidi(0x08FF));
2490 assert!(is_utf16_code_unit_bidi(0x061C));
2491 assert!(is_utf16_code_unit_bidi(0xFB1D));
2492 assert!(is_utf16_code_unit_bidi(0xFB50));
2493 assert!(is_utf16_code_unit_bidi(0xFDFF));
2494 assert!(is_utf16_code_unit_bidi(0xFE70));
2495 assert!(is_utf16_code_unit_bidi(0xFEFE));
2496 assert!(is_utf16_code_unit_bidi(0x200F));
2497 assert!(is_utf16_code_unit_bidi(0x202B));
2498 assert!(is_utf16_code_unit_bidi(0x202E));
2499 assert!(is_utf16_code_unit_bidi(0x2067));
2500 assert!(is_utf16_code_unit_bidi(0xD802));
2501 assert!(is_utf16_code_unit_bidi(0xD803));
2502 assert!(is_utf16_code_unit_bidi(0xD83A));
2503 assert!(is_utf16_code_unit_bidi(0xD83B));
2504 }
2505
2506 #[test]
2507 fn test_is_str_bidi() {
2508 assert!(!is_str_bidi("abcdefghijklmnopaabcdefghijklmnop"));
2509 assert!(!is_str_bidi("abcdefghijklmnop\u{03B1}abcdefghijklmnop"));
2510 assert!(!is_str_bidi("abcdefghijklmnop\u{3041}abcdefghijklmnop"));
2511 assert!(!is_str_bidi("abcdefghijklmnop\u{1F4A9}abcdefghijklmnop"));
2512 assert!(!is_str_bidi("abcdefghijklmnop\u{FE00}abcdefghijklmnop"));
2513 assert!(!is_str_bidi("abcdefghijklmnop\u{202C}abcdefghijklmnop"));
2514 assert!(!is_str_bidi("abcdefghijklmnop\u{FEFF}abcdefghijklmnop"));
2515 assert!(is_str_bidi("abcdefghijklmnop\u{0590}abcdefghijklmnop"));
2516 assert!(is_str_bidi("abcdefghijklmnop\u{08FF}abcdefghijklmnop"));
2517 assert!(is_str_bidi("abcdefghijklmnop\u{061C}abcdefghijklmnop"));
2518 assert!(is_str_bidi("abcdefghijklmnop\u{FB50}abcdefghijklmnop"));
2519 assert!(is_str_bidi("abcdefghijklmnop\u{FDFF}abcdefghijklmnop"));
2520 assert!(is_str_bidi("abcdefghijklmnop\u{FE70}abcdefghijklmnop"));
2521 assert!(is_str_bidi("abcdefghijklmnop\u{FEFE}abcdefghijklmnop"));
2522 assert!(is_str_bidi("abcdefghijklmnop\u{200F}abcdefghijklmnop"));
2523 assert!(is_str_bidi("abcdefghijklmnop\u{202B}abcdefghijklmnop"));
2524 assert!(is_str_bidi("abcdefghijklmnop\u{202E}abcdefghijklmnop"));
2525 assert!(is_str_bidi("abcdefghijklmnop\u{2067}abcdefghijklmnop"));
2526 assert!(is_str_bidi("abcdefghijklmnop\u{10800}abcdefghijklmnop"));
2527 assert!(is_str_bidi("abcdefghijklmnop\u{10FFF}abcdefghijklmnop"));
2528 assert!(is_str_bidi("abcdefghijklmnop\u{1E800}abcdefghijklmnop"));
2529 assert!(is_str_bidi("abcdefghijklmnop\u{1EFFF}abcdefghijklmnop"));
2530 }
2531
2532 #[test]
2533 fn test_is_utf8_bidi() {
2534 assert!(!is_utf8_bidi(
2535 "abcdefghijklmnopaabcdefghijklmnop".as_bytes()
2536 ));
2537 assert!(!is_utf8_bidi(
2538 "abcdefghijklmnop\u{03B1}abcdefghijklmnop".as_bytes()
2539 ));
2540 assert!(!is_utf8_bidi(
2541 "abcdefghijklmnop\u{3041}abcdefghijklmnop".as_bytes()
2542 ));
2543 assert!(!is_utf8_bidi(
2544 "abcdefghijklmnop\u{1F4A9}abcdefghijklmnop".as_bytes()
2545 ));
2546 assert!(!is_utf8_bidi(
2547 "abcdefghijklmnop\u{FE00}abcdefghijklmnop".as_bytes()
2548 ));
2549 assert!(!is_utf8_bidi(
2550 "abcdefghijklmnop\u{202C}abcdefghijklmnop".as_bytes()
2551 ));
2552 assert!(!is_utf8_bidi(
2553 "abcdefghijklmnop\u{FEFF}abcdefghijklmnop".as_bytes()
2554 ));
2555 assert!(is_utf8_bidi(
2556 "abcdefghijklmnop\u{0590}abcdefghijklmnop".as_bytes()
2557 ));
2558 assert!(is_utf8_bidi(
2559 "abcdefghijklmnop\u{08FF}abcdefghijklmnop".as_bytes()
2560 ));
2561 assert!(is_utf8_bidi(
2562 "abcdefghijklmnop\u{061C}abcdefghijklmnop".as_bytes()
2563 ));
2564 assert!(is_utf8_bidi(
2565 "abcdefghijklmnop\u{FB50}abcdefghijklmnop".as_bytes()
2566 ));
2567 assert!(is_utf8_bidi(
2568 "abcdefghijklmnop\u{FDFF}abcdefghijklmnop".as_bytes()
2569 ));
2570 assert!(is_utf8_bidi(
2571 "abcdefghijklmnop\u{FE70}abcdefghijklmnop".as_bytes()
2572 ));
2573 assert!(is_utf8_bidi(
2574 "abcdefghijklmnop\u{FEFE}abcdefghijklmnop".as_bytes()
2575 ));
2576 assert!(is_utf8_bidi(
2577 "abcdefghijklmnop\u{200F}abcdefghijklmnop".as_bytes()
2578 ));
2579 assert!(is_utf8_bidi(
2580 "abcdefghijklmnop\u{202B}abcdefghijklmnop".as_bytes()
2581 ));
2582 assert!(is_utf8_bidi(
2583 "abcdefghijklmnop\u{202E}abcdefghijklmnop".as_bytes()
2584 ));
2585 assert!(is_utf8_bidi(
2586 "abcdefghijklmnop\u{2067}abcdefghijklmnop".as_bytes()
2587 ));
2588 assert!(is_utf8_bidi(
2589 "abcdefghijklmnop\u{10800}abcdefghijklmnop".as_bytes()
2590 ));
2591 assert!(is_utf8_bidi(
2592 "abcdefghijklmnop\u{10FFF}abcdefghijklmnop".as_bytes()
2593 ));
2594 assert!(is_utf8_bidi(
2595 "abcdefghijklmnop\u{1E800}abcdefghijklmnop".as_bytes()
2596 ));
2597 assert!(is_utf8_bidi(
2598 "abcdefghijklmnop\u{1EFFF}abcdefghijklmnop".as_bytes()
2599 ));
2600 }
2601
2602 #[test]
2603 fn test_is_utf16_bidi() {
2604 assert!(!is_utf16_bidi(&[
2605 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x0062, 0x62, 0x63, 0x64, 0x65, 0x66,
2606 0x67, 0x68, 0x69,
2607 ]));
2608 assert!(!is_utf16_bidi(&[
2609 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x03B1, 0x62, 0x63, 0x64, 0x65, 0x66,
2610 0x67, 0x68, 0x69,
2611 ]));
2612 assert!(!is_utf16_bidi(&[
2613 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x3041, 0x62, 0x63, 0x64, 0x65, 0x66,
2614 0x67, 0x68, 0x69,
2615 ]));
2616 assert!(!is_utf16_bidi(&[
2617 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD801, 0x62, 0x63, 0x64, 0x65, 0x66,
2618 0x67, 0x68, 0x69,
2619 ]));
2620 assert!(!is_utf16_bidi(&[
2621 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFE00, 0x62, 0x63, 0x64, 0x65, 0x66,
2622 0x67, 0x68, 0x69,
2623 ]));
2624 assert!(!is_utf16_bidi(&[
2625 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x202C, 0x62, 0x63, 0x64, 0x65, 0x66,
2626 0x67, 0x68, 0x69,
2627 ]));
2628 assert!(!is_utf16_bidi(&[
2629 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFEFF, 0x62, 0x63, 0x64, 0x65, 0x66,
2630 0x67, 0x68, 0x69,
2631 ]));
2632 assert!(is_utf16_bidi(&[
2633 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x0590, 0x62, 0x63, 0x64, 0x65, 0x66,
2634 0x67, 0x68, 0x69,
2635 ]));
2636 assert!(is_utf16_bidi(&[
2637 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x08FF, 0x62, 0x63, 0x64, 0x65, 0x66,
2638 0x67, 0x68, 0x69,
2639 ]));
2640 assert!(is_utf16_bidi(&[
2641 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x061C, 0x62, 0x63, 0x64, 0x65, 0x66,
2642 0x67, 0x68, 0x69,
2643 ]));
2644 assert!(is_utf16_bidi(&[
2645 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFB1D, 0x62, 0x63, 0x64, 0x65, 0x66,
2646 0x67, 0x68, 0x69,
2647 ]));
2648 assert!(is_utf16_bidi(&[
2649 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFB50, 0x62, 0x63, 0x64, 0x65, 0x66,
2650 0x67, 0x68, 0x69,
2651 ]));
2652 assert!(is_utf16_bidi(&[
2653 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFDFF, 0x62, 0x63, 0x64, 0x65, 0x66,
2654 0x67, 0x68, 0x69,
2655 ]));
2656 assert!(is_utf16_bidi(&[
2657 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFE70, 0x62, 0x63, 0x64, 0x65, 0x66,
2658 0x67, 0x68, 0x69,
2659 ]));
2660 assert!(is_utf16_bidi(&[
2661 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFEFE, 0x62, 0x63, 0x64, 0x65, 0x66,
2662 0x67, 0x68, 0x69,
2663 ]));
2664 assert!(is_utf16_bidi(&[
2665 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x200F, 0x62, 0x63, 0x64, 0x65, 0x66,
2666 0x67, 0x68, 0x69,
2667 ]));
2668 assert!(is_utf16_bidi(&[
2669 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x202B, 0x62, 0x63, 0x64, 0x65, 0x66,
2670 0x67, 0x68, 0x69,
2671 ]));
2672 assert!(is_utf16_bidi(&[
2673 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x202E, 0x62, 0x63, 0x64, 0x65, 0x66,
2674 0x67, 0x68, 0x69,
2675 ]));
2676 assert!(is_utf16_bidi(&[
2677 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x2067, 0x62, 0x63, 0x64, 0x65, 0x66,
2678 0x67, 0x68, 0x69,
2679 ]));
2680 assert!(is_utf16_bidi(&[
2681 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD802, 0x62, 0x63, 0x64, 0x65, 0x66,
2682 0x67, 0x68, 0x69,
2683 ]));
2684 assert!(is_utf16_bidi(&[
2685 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD803, 0x62, 0x63, 0x64, 0x65, 0x66,
2686 0x67, 0x68, 0x69,
2687 ]));
2688 assert!(is_utf16_bidi(&[
2689 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD83A, 0x62, 0x63, 0x64, 0x65, 0x66,
2690 0x67, 0x68, 0x69,
2691 ]));
2692 assert!(is_utf16_bidi(&[
2693 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD83B, 0x62, 0x63, 0x64, 0x65, 0x66,
2694 0x67, 0x68, 0x69,
2695 ]));
2696
2697 assert!(is_utf16_bidi(&[
2698 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x0590, 0x3041, 0x62, 0x63, 0x64, 0x65,
2699 0x66, 0x67, 0x68, 0x69,
2700 ]));
2701 }
2702
2703 #[test]
2704 fn test_check_str_for_latin1_and_bidi() {
2705 assert_ne!(
2706 check_str_for_latin1_and_bidi("abcdefghijklmnopaabcdefghijklmnop"),
2707 Latin1Bidi::Bidi
2708 );
2709 assert_ne!(
2710 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{03B1}abcdefghijklmnop"),
2711 Latin1Bidi::Bidi
2712 );
2713 assert_ne!(
2714 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{3041}abcdefghijklmnop"),
2715 Latin1Bidi::Bidi
2716 );
2717 assert_ne!(
2718 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{1F4A9}abcdefghijklmnop"),
2719 Latin1Bidi::Bidi
2720 );
2721 assert_ne!(
2722 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{FE00}abcdefghijklmnop"),
2723 Latin1Bidi::Bidi
2724 );
2725 assert_ne!(
2726 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{202C}abcdefghijklmnop"),
2727 Latin1Bidi::Bidi
2728 );
2729 assert_ne!(
2730 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{FEFF}abcdefghijklmnop"),
2731 Latin1Bidi::Bidi
2732 );
2733 assert_eq!(
2734 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{0590}abcdefghijklmnop"),
2735 Latin1Bidi::Bidi
2736 );
2737 assert_eq!(
2738 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{08FF}abcdefghijklmnop"),
2739 Latin1Bidi::Bidi
2740 );
2741 assert_eq!(
2742 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{061C}abcdefghijklmnop"),
2743 Latin1Bidi::Bidi
2744 );
2745 assert_eq!(
2746 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{FB50}abcdefghijklmnop"),
2747 Latin1Bidi::Bidi
2748 );
2749 assert_eq!(
2750 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{FDFF}abcdefghijklmnop"),
2751 Latin1Bidi::Bidi
2752 );
2753 assert_eq!(
2754 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{FE70}abcdefghijklmnop"),
2755 Latin1Bidi::Bidi
2756 );
2757 assert_eq!(
2758 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{FEFE}abcdefghijklmnop"),
2759 Latin1Bidi::Bidi
2760 );
2761 assert_eq!(
2762 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{200F}abcdefghijklmnop"),
2763 Latin1Bidi::Bidi
2764 );
2765 assert_eq!(
2766 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{202B}abcdefghijklmnop"),
2767 Latin1Bidi::Bidi
2768 );
2769 assert_eq!(
2770 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{202E}abcdefghijklmnop"),
2771 Latin1Bidi::Bidi
2772 );
2773 assert_eq!(
2774 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{2067}abcdefghijklmnop"),
2775 Latin1Bidi::Bidi
2776 );
2777 assert_eq!(
2778 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{10800}abcdefghijklmnop"),
2779 Latin1Bidi::Bidi
2780 );
2781 assert_eq!(
2782 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{10FFF}abcdefghijklmnop"),
2783 Latin1Bidi::Bidi
2784 );
2785 assert_eq!(
2786 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{1E800}abcdefghijklmnop"),
2787 Latin1Bidi::Bidi
2788 );
2789 assert_eq!(
2790 check_str_for_latin1_and_bidi("abcdefghijklmnop\u{1EFFF}abcdefghijklmnop"),
2791 Latin1Bidi::Bidi
2792 );
2793 }
2794
2795 #[test]
2796 fn test_check_utf8_for_latin1_and_bidi() {
2797 assert_ne!(
2798 check_utf8_for_latin1_and_bidi("abcdefghijklmnopaabcdefghijklmnop".as_bytes()),
2799 Latin1Bidi::Bidi
2800 );
2801 assert_ne!(
2802 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{03B1}abcdefghijklmnop".as_bytes()),
2803 Latin1Bidi::Bidi
2804 );
2805 assert_ne!(
2806 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{3041}abcdefghijklmnop".as_bytes()),
2807 Latin1Bidi::Bidi
2808 );
2809 assert_ne!(
2810 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{1F4A9}abcdefghijklmnop".as_bytes()),
2811 Latin1Bidi::Bidi
2812 );
2813 assert_ne!(
2814 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{FE00}abcdefghijklmnop".as_bytes()),
2815 Latin1Bidi::Bidi
2816 );
2817 assert_ne!(
2818 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{202C}abcdefghijklmnop".as_bytes()),
2819 Latin1Bidi::Bidi
2820 );
2821 assert_ne!(
2822 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{FEFF}abcdefghijklmnop".as_bytes()),
2823 Latin1Bidi::Bidi
2824 );
2825 assert_eq!(
2826 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{0590}abcdefghijklmnop".as_bytes()),
2827 Latin1Bidi::Bidi
2828 );
2829 assert_eq!(
2830 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{08FF}abcdefghijklmnop".as_bytes()),
2831 Latin1Bidi::Bidi
2832 );
2833 assert_eq!(
2834 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{061C}abcdefghijklmnop".as_bytes()),
2835 Latin1Bidi::Bidi
2836 );
2837 assert_eq!(
2838 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{FB50}abcdefghijklmnop".as_bytes()),
2839 Latin1Bidi::Bidi
2840 );
2841 assert_eq!(
2842 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{FDFF}abcdefghijklmnop".as_bytes()),
2843 Latin1Bidi::Bidi
2844 );
2845 assert_eq!(
2846 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{FE70}abcdefghijklmnop".as_bytes()),
2847 Latin1Bidi::Bidi
2848 );
2849 assert_eq!(
2850 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{FEFE}abcdefghijklmnop".as_bytes()),
2851 Latin1Bidi::Bidi
2852 );
2853 assert_eq!(
2854 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{200F}abcdefghijklmnop".as_bytes()),
2855 Latin1Bidi::Bidi
2856 );
2857 assert_eq!(
2858 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{202B}abcdefghijklmnop".as_bytes()),
2859 Latin1Bidi::Bidi
2860 );
2861 assert_eq!(
2862 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{202E}abcdefghijklmnop".as_bytes()),
2863 Latin1Bidi::Bidi
2864 );
2865 assert_eq!(
2866 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{2067}abcdefghijklmnop".as_bytes()),
2867 Latin1Bidi::Bidi
2868 );
2869 assert_eq!(
2870 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{10800}abcdefghijklmnop".as_bytes()),
2871 Latin1Bidi::Bidi
2872 );
2873 assert_eq!(
2874 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{10FFF}abcdefghijklmnop".as_bytes()),
2875 Latin1Bidi::Bidi
2876 );
2877 assert_eq!(
2878 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{1E800}abcdefghijklmnop".as_bytes()),
2879 Latin1Bidi::Bidi
2880 );
2881 assert_eq!(
2882 check_utf8_for_latin1_and_bidi("abcdefghijklmnop\u{1EFFF}abcdefghijklmnop".as_bytes()),
2883 Latin1Bidi::Bidi
2884 );
2885 }
2886
2887 #[test]
2888 fn test_check_utf16_for_latin1_and_bidi() {
2889 assert_ne!(
2890 check_utf16_for_latin1_and_bidi(&[
2891 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x0062, 0x62, 0x63, 0x64, 0x65,
2892 0x66, 0x67, 0x68, 0x69,
2893 ]),
2894 Latin1Bidi::Bidi
2895 );
2896 assert_ne!(
2897 check_utf16_for_latin1_and_bidi(&[
2898 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x03B1, 0x62, 0x63, 0x64, 0x65,
2899 0x66, 0x67, 0x68, 0x69,
2900 ]),
2901 Latin1Bidi::Bidi
2902 );
2903 assert_ne!(
2904 check_utf16_for_latin1_and_bidi(&[
2905 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x3041, 0x62, 0x63, 0x64, 0x65,
2906 0x66, 0x67, 0x68, 0x69,
2907 ]),
2908 Latin1Bidi::Bidi
2909 );
2910 assert_ne!(
2911 check_utf16_for_latin1_and_bidi(&[
2912 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD801, 0x62, 0x63, 0x64, 0x65,
2913 0x66, 0x67, 0x68, 0x69,
2914 ]),
2915 Latin1Bidi::Bidi
2916 );
2917 assert_ne!(
2918 check_utf16_for_latin1_and_bidi(&[
2919 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFE00, 0x62, 0x63, 0x64, 0x65,
2920 0x66, 0x67, 0x68, 0x69,
2921 ]),
2922 Latin1Bidi::Bidi
2923 );
2924 assert_ne!(
2925 check_utf16_for_latin1_and_bidi(&[
2926 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x202C, 0x62, 0x63, 0x64, 0x65,
2927 0x66, 0x67, 0x68, 0x69,
2928 ]),
2929 Latin1Bidi::Bidi
2930 );
2931 assert_ne!(
2932 check_utf16_for_latin1_and_bidi(&[
2933 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFEFF, 0x62, 0x63, 0x64, 0x65,
2934 0x66, 0x67, 0x68, 0x69,
2935 ]),
2936 Latin1Bidi::Bidi
2937 );
2938 assert_eq!(
2939 check_utf16_for_latin1_and_bidi(&[
2940 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x0590, 0x62, 0x63, 0x64, 0x65,
2941 0x66, 0x67, 0x68, 0x69,
2942 ]),
2943 Latin1Bidi::Bidi
2944 );
2945 assert_eq!(
2946 check_utf16_for_latin1_and_bidi(&[
2947 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x08FF, 0x62, 0x63, 0x64, 0x65,
2948 0x66, 0x67, 0x68, 0x69,
2949 ]),
2950 Latin1Bidi::Bidi
2951 );
2952 assert_eq!(
2953 check_utf16_for_latin1_and_bidi(&[
2954 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x061C, 0x62, 0x63, 0x64, 0x65,
2955 0x66, 0x67, 0x68, 0x69,
2956 ]),
2957 Latin1Bidi::Bidi
2958 );
2959 assert_eq!(
2960 check_utf16_for_latin1_and_bidi(&[
2961 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFB1D, 0x62, 0x63, 0x64, 0x65,
2962 0x66, 0x67, 0x68, 0x69,
2963 ]),
2964 Latin1Bidi::Bidi
2965 );
2966 assert_eq!(
2967 check_utf16_for_latin1_and_bidi(&[
2968 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFB50, 0x62, 0x63, 0x64, 0x65,
2969 0x66, 0x67, 0x68, 0x69,
2970 ]),
2971 Latin1Bidi::Bidi
2972 );
2973 assert_eq!(
2974 check_utf16_for_latin1_and_bidi(&[
2975 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFDFF, 0x62, 0x63, 0x64, 0x65,
2976 0x66, 0x67, 0x68, 0x69,
2977 ]),
2978 Latin1Bidi::Bidi
2979 );
2980 assert_eq!(
2981 check_utf16_for_latin1_and_bidi(&[
2982 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFE70, 0x62, 0x63, 0x64, 0x65,
2983 0x66, 0x67, 0x68, 0x69,
2984 ]),
2985 Latin1Bidi::Bidi
2986 );
2987 assert_eq!(
2988 check_utf16_for_latin1_and_bidi(&[
2989 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xFEFE, 0x62, 0x63, 0x64, 0x65,
2990 0x66, 0x67, 0x68, 0x69,
2991 ]),
2992 Latin1Bidi::Bidi
2993 );
2994 assert_eq!(
2995 check_utf16_for_latin1_and_bidi(&[
2996 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x200F, 0x62, 0x63, 0x64, 0x65,
2997 0x66, 0x67, 0x68, 0x69,
2998 ]),
2999 Latin1Bidi::Bidi
3000 );
3001 assert_eq!(
3002 check_utf16_for_latin1_and_bidi(&[
3003 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x202B, 0x62, 0x63, 0x64, 0x65,
3004 0x66, 0x67, 0x68, 0x69,
3005 ]),
3006 Latin1Bidi::Bidi
3007 );
3008 assert_eq!(
3009 check_utf16_for_latin1_and_bidi(&[
3010 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x202E, 0x62, 0x63, 0x64, 0x65,
3011 0x66, 0x67, 0x68, 0x69,
3012 ]),
3013 Latin1Bidi::Bidi
3014 );
3015 assert_eq!(
3016 check_utf16_for_latin1_and_bidi(&[
3017 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x2067, 0x62, 0x63, 0x64, 0x65,
3018 0x66, 0x67, 0x68, 0x69,
3019 ]),
3020 Latin1Bidi::Bidi
3021 );
3022 assert_eq!(
3023 check_utf16_for_latin1_and_bidi(&[
3024 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD802, 0x62, 0x63, 0x64, 0x65,
3025 0x66, 0x67, 0x68, 0x69,
3026 ]),
3027 Latin1Bidi::Bidi
3028 );
3029 assert_eq!(
3030 check_utf16_for_latin1_and_bidi(&[
3031 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD803, 0x62, 0x63, 0x64, 0x65,
3032 0x66, 0x67, 0x68, 0x69,
3033 ]),
3034 Latin1Bidi::Bidi
3035 );
3036 assert_eq!(
3037 check_utf16_for_latin1_and_bidi(&[
3038 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD83A, 0x62, 0x63, 0x64, 0x65,
3039 0x66, 0x67, 0x68, 0x69,
3040 ]),
3041 Latin1Bidi::Bidi
3042 );
3043 assert_eq!(
3044 check_utf16_for_latin1_and_bidi(&[
3045 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xD83B, 0x62, 0x63, 0x64, 0x65,
3046 0x66, 0x67, 0x68, 0x69,
3047 ]),
3048 Latin1Bidi::Bidi
3049 );
3050
3051 assert_eq!(
3052 check_utf16_for_latin1_and_bidi(&[
3053 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x0590, 0x3041, 0x62, 0x63, 0x64,
3054 0x65, 0x66, 0x67, 0x68, 0x69,
3055 ]),
3056 Latin1Bidi::Bidi
3057 );
3058 }
3059
3060 #[inline(always)]
3061 pub fn reference_is_char_bidi(c: char) -> bool {
3062 match c {
3063 '\u{0590}'..='\u{08FF}'
3064 | '\u{FB1D}'..='\u{FDFF}'
3065 | '\u{FE70}'..='\u{FEFE}'
3066 | '\u{10800}'..='\u{10FFF}'
3067 | '\u{1E800}'..='\u{1EFFF}'
3068 | '\u{200F}'
3069 | '\u{202B}'
3070 | '\u{202E}'
3071 | '\u{2067}' => true,
3072 _ => false,
3073 }
3074 }
3075
3076 #[inline(always)]
3077 pub fn reference_is_utf16_code_unit_bidi(u: u16) -> bool {
3078 match u {
3079 0x0590..=0x08FF
3080 | 0xFB1D..=0xFDFF
3081 | 0xFE70..=0xFEFE
3082 | 0xD802
3083 | 0xD803
3084 | 0xD83A
3085 | 0xD83B
3086 | 0x200F
3087 | 0x202B
3088 | 0x202E
3089 | 0x2067 => true,
3090 _ => false,
3091 }
3092 }
3093
3094 #[test]
3095 #[cfg_attr(miri, ignore)] fn test_is_char_bidi_thoroughly() {
3097 for i in 0..0xD800u32 {
3098 let c: char = ::core::char::from_u32(i).unwrap();
3099 assert_eq!(is_char_bidi(c), reference_is_char_bidi(c));
3100 }
3101 for i in 0xE000..0x110000u32 {
3102 let c: char = ::core::char::from_u32(i).unwrap();
3103 assert_eq!(is_char_bidi(c), reference_is_char_bidi(c));
3104 }
3105 }
3106
3107 #[test]
3108 #[cfg_attr(miri, ignore)] fn test_is_utf16_code_unit_bidi_thoroughly() {
3110 for i in 0..0x10000u32 {
3111 let u = i as u16;
3112 assert_eq!(
3113 is_utf16_code_unit_bidi(u),
3114 reference_is_utf16_code_unit_bidi(u)
3115 );
3116 }
3117 }
3118
3119 #[test]
3120 #[cfg_attr(miri, ignore)] fn test_is_str_bidi_thoroughly() {
3122 let mut buf = [0; 4];
3123 for i in 0..0xD800u32 {
3124 let c: char = ::core::char::from_u32(i).unwrap();
3125 assert_eq!(
3126 is_str_bidi(c.encode_utf8(&mut buf[..])),
3127 reference_is_char_bidi(c)
3128 );
3129 }
3130 for i in 0xE000..0x110000u32 {
3131 let c: char = ::core::char::from_u32(i).unwrap();
3132 assert_eq!(
3133 is_str_bidi(c.encode_utf8(&mut buf[..])),
3134 reference_is_char_bidi(c)
3135 );
3136 }
3137 }
3138
3139 #[test]
3140 #[cfg_attr(miri, ignore)] fn test_is_utf8_bidi_thoroughly() {
3142 let mut buf = [0; 8];
3143 for i in 0..0xD800u32 {
3144 let c: char = ::core::char::from_u32(i).unwrap();
3145 let expect = reference_is_char_bidi(c);
3146 {
3147 let len = {
3148 let bytes = c.encode_utf8(&mut buf[..]).as_bytes();
3149 assert_eq!(is_utf8_bidi(bytes), expect);
3150 bytes.len()
3151 };
3152 {
3153 let tail = &mut buf[len..];
3154 for b in tail.iter_mut() {
3155 *b = 0;
3156 }
3157 }
3158 }
3159 assert_eq!(is_utf8_bidi(&buf[..]), expect);
3160 }
3161 for i in 0xE000..0x110000u32 {
3162 let c: char = ::core::char::from_u32(i).unwrap();
3163 let expect = reference_is_char_bidi(c);
3164 {
3165 let len = {
3166 let bytes = c.encode_utf8(&mut buf[..]).as_bytes();
3167 assert_eq!(is_utf8_bidi(bytes), expect);
3168 bytes.len()
3169 };
3170 {
3171 let tail = &mut buf[len..];
3172 for b in tail.iter_mut() {
3173 *b = 0;
3174 }
3175 }
3176 }
3177 assert_eq!(is_utf8_bidi(&buf[..]), expect);
3178 }
3179 }
3180
3181 #[test]
3182 #[cfg_attr(miri, ignore)] fn test_is_utf16_bidi_thoroughly() {
3184 let mut buf = [0; 32];
3185 for i in 0..0x10000u32 {
3186 let u = i as u16;
3187 buf[15] = u;
3188 assert_eq!(
3189 is_utf16_bidi(&buf[..]),
3190 reference_is_utf16_code_unit_bidi(u)
3191 );
3192 }
3193 }
3194
3195 #[test]
3196 fn test_is_utf8_bidi_edge_cases() {
3197 assert!(!is_utf8_bidi(b"\xD5\xBF\x61"));
3198 assert!(!is_utf8_bidi(b"\xD6\x80\x61"));
3199 assert!(!is_utf8_bidi(b"abc"));
3200 assert!(is_utf8_bidi(b"\xD5\xBF\xC2"));
3201 assert!(is_utf8_bidi(b"\xD6\x80\xC2"));
3202 assert!(is_utf8_bidi(b"ab\xC2"));
3203 }
3204
3205 #[test]
3206 fn test_decode_latin1() {
3207 match decode_latin1(b"ab") {
3208 Cow::Borrowed(s) => {
3209 assert_eq!(s, "ab");
3210 }
3211 Cow::Owned(_) => {
3212 unreachable!("Should have borrowed");
3213 }
3214 }
3215 assert_eq!(decode_latin1(b"a\xE4"), "a\u{E4}");
3216 }
3217
3218 #[test]
3219 fn test_encode_latin1_lossy() {
3220 match encode_latin1_lossy("ab") {
3221 Cow::Borrowed(s) => {
3222 assert_eq!(s, b"ab");
3223 }
3224 Cow::Owned(_) => {
3225 unreachable!("Should have borrowed");
3226 }
3227 }
3228 assert_eq!(encode_latin1_lossy("a\u{E4}"), &(b"a\xE4")[..]);
3229 }
3230
3231 #[test]
3232 fn test_convert_utf8_to_utf16_without_replacement() {
3233 let mut buf = [0u16; 5];
3234 assert_eq!(
3235 convert_utf8_to_utf16_without_replacement(b"ab", &mut buf[..2]),
3236 Some(2)
3237 );
3238 assert_eq!(buf[0], u16::from(b'a'));
3239 assert_eq!(buf[1], u16::from(b'b'));
3240 assert_eq!(buf[2], 0);
3241 assert_eq!(
3242 convert_utf8_to_utf16_without_replacement(b"\xC3\xA4c", &mut buf[..3]),
3243 Some(2)
3244 );
3245 assert_eq!(buf[0], 0xE4);
3246 assert_eq!(buf[1], u16::from(b'c'));
3247 assert_eq!(buf[2], 0);
3248 assert_eq!(
3249 convert_utf8_to_utf16_without_replacement(b"\xE2\x98\x83", &mut buf[..3]),
3250 Some(1)
3251 );
3252 assert_eq!(buf[0], 0x2603);
3253 assert_eq!(buf[1], u16::from(b'c'));
3254 assert_eq!(buf[2], 0);
3255 assert_eq!(
3256 convert_utf8_to_utf16_without_replacement(b"\xE2\x98\x83d", &mut buf[..4]),
3257 Some(2)
3258 );
3259 assert_eq!(buf[0], 0x2603);
3260 assert_eq!(buf[1], u16::from(b'd'));
3261 assert_eq!(buf[2], 0);
3262 assert_eq!(
3263 convert_utf8_to_utf16_without_replacement(b"\xE2\x98\x83\xC3\xA4", &mut buf[..5]),
3264 Some(2)
3265 );
3266 assert_eq!(buf[0], 0x2603);
3267 assert_eq!(buf[1], 0xE4);
3268 assert_eq!(buf[2], 0);
3269 assert_eq!(
3270 convert_utf8_to_utf16_without_replacement(b"\xF0\x9F\x93\x8E", &mut buf[..4]),
3271 Some(2)
3272 );
3273 assert_eq!(buf[0], 0xD83D);
3274 assert_eq!(buf[1], 0xDCCE);
3275 assert_eq!(buf[2], 0);
3276 assert_eq!(
3277 convert_utf8_to_utf16_without_replacement(b"\xF0\x9F\x93\x8Ee", &mut buf[..5]),
3278 Some(3)
3279 );
3280 assert_eq!(buf[0], 0xD83D);
3281 assert_eq!(buf[1], 0xDCCE);
3282 assert_eq!(buf[2], u16::from(b'e'));
3283 assert_eq!(
3284 convert_utf8_to_utf16_without_replacement(b"\xF0\x9F\x93", &mut buf[..5]),
3285 None
3286 );
3287 }
3288}