1use core::hint::black_box;
10
11const LO_U64: u64 = 0x0101_0101_0101_0101;
12const HI_U64: u64 = 0x8080_8080_8080_8080;
13
14#[inline(always)]
15fn has_zero_byte(word: u64) -> u64 {
16 word.wrapping_sub(LO_U64) & !word & HI_U64
17}
18
19#[inline(always)]
20fn first_match_index_le(mask: u64) -> usize {
21 (mask.trailing_zeros() >> 3) as usize
22}
23
24#[inline(always)]
25fn last_match_index_le(mask: u64) -> usize {
26 7 - ((mask.leading_zeros() >> 3) as usize)
27}
28
29#[inline(always)]
30unsafe fn read_u64_aligned(ptr: *const u8) -> u64 {
31 debug_assert_eq!((ptr as usize) & 7, 0);
32 unsafe { *ptr.cast::<u64>() }
33}
34
35#[inline(always)]
36unsafe fn write_u64_aligned(ptr: *mut u8, val: u64) {
37 debug_assert_eq!((ptr as usize) & 7, 0);
38 unsafe {
39 *ptr.cast::<u64>() = val;
40 }
41}
42
43#[cold]
44#[inline(never)]
45unsafe fn copy_forward_misaligned(
46 dst: *mut u8,
47 src: *const u8,
48 mut i: usize,
49 n: usize,
50 src_off: usize,
51) -> usize {
52 if n - i >= 16 {
53 let k = 8 - src_off;
54 let rshift = (src_off * 8) as u32;
55 let lshift = (k * 8) as u32;
56 let mut low: u64 = 0;
57 let mut b = 0;
58 while b < k {
59 low |= (unsafe { *src.add(i + b) } as u64) << (b * 8);
60 b += 1;
61 }
62
63 while n - i >= 40 {
64 unsafe {
65 let s_al = src.add(i + k);
66 let d_al = dst.add(i);
67 let w0 = u64::from_le(read_u64_aligned(s_al));
68 let w1 = u64::from_le(read_u64_aligned(s_al.add(8)));
69 let w2 = u64::from_le(read_u64_aligned(s_al.add(16)));
70 let w3 = u64::from_le(read_u64_aligned(s_al.add(24)));
71 write_u64_aligned(d_al, (low | (w0 << lshift)).to_le());
72 write_u64_aligned(d_al.add(8), ((w0 >> rshift) | (w1 << lshift)).to_le());
73 write_u64_aligned(d_al.add(16), ((w1 >> rshift) | (w2 << lshift)).to_le());
74 write_u64_aligned(d_al.add(24), ((w2 >> rshift) | (w3 << lshift)).to_le());
75 low = w3 >> rshift;
76 }
77 i += 32;
78 }
79
80 while n - i >= 16 {
81 unsafe {
82 let w0 = u64::from_le(read_u64_aligned(src.add(i + k)));
83 write_u64_aligned(dst.add(i), (low | (w0 << lshift)).to_le());
84 low = w0 >> rshift;
85 }
86 i += 8;
87 }
88 }
89 i
90}
91
92pub unsafe fn copy_nonoverlapping_bytes(dst: *mut u8, src: *const u8, n: usize) -> *mut u8 {
99 let dst = black_box(dst);
100 let src = black_box(src);
101 let mut i = 0;
102
103 if n >= 16 {
104 let misalign = (dst as usize) & 15;
105 if misalign != 0 {
106 let prolog = (16 - misalign) & 15;
107 while i < prolog {
108 unsafe {
109 *dst.add(i) = black_box(*src.add(i));
110 }
111 i += 1;
112 }
113 }
114
115 let src_off = (unsafe { src.add(i) } as usize) & 7;
116 if src_off == 0 {
117 while n - i >= 64 {
118 unsafe {
119 let s = src.add(i).cast::<u64>();
120 let d = dst.add(i).cast::<u64>();
121 let w0 = black_box(*s);
122 let w1 = *s.add(1);
123 let w2 = *s.add(2);
124 let w3 = *s.add(3);
125 let w4 = *s.add(4);
126 let w5 = *s.add(5);
127 let w6 = *s.add(6);
128 let w7 = *s.add(7);
129 *d = w0;
130 *d.add(1) = w1;
131 *d.add(2) = w2;
132 *d.add(3) = w3;
133 *d.add(4) = w4;
134 *d.add(5) = w5;
135 *d.add(6) = w6;
136 *d.add(7) = w7;
137 }
138 i += 64;
139 }
140
141 if n - i >= 32 {
142 unsafe {
143 let s = src.add(i);
144 let d = dst.add(i);
145 let w0 = read_u64_aligned(s);
146 let w1 = read_u64_aligned(s.add(8));
147 let w2 = read_u64_aligned(s.add(16));
148 let w3 = read_u64_aligned(s.add(24));
149 write_u64_aligned(d, w0);
150 write_u64_aligned(d.add(8), w1);
151 write_u64_aligned(d.add(16), w2);
152 write_u64_aligned(d.add(24), w3);
153 }
154 i += 32;
155 }
156
157 if n - i >= 16 {
158 unsafe {
159 let s = src.add(i);
160 let d = dst.add(i);
161 let w0 = read_u64_aligned(s);
162 let w1 = read_u64_aligned(s.add(8));
163 write_u64_aligned(d, w0);
164 write_u64_aligned(d.add(8), w1);
165 }
166 i += 16;
167 }
168
169 if n - i >= 8 {
170 unsafe {
171 let w = read_u64_aligned(src.add(i));
172 write_u64_aligned(dst.add(i), w);
173 }
174 i += 8;
175 }
176 } else {
177 i = unsafe { copy_forward_misaligned(dst, src, i, n, src_off) };
178 }
179 }
180
181 while i < n {
182 unsafe {
183 *dst.add(i) = black_box(*src.add(i));
184 }
185 i += 1;
186 }
187
188 dst
189}
190
191pub unsafe fn copy_bytes(dst: *mut u8, src: *const u8, n: usize) -> *mut u8 {
197 let d_addr = dst as usize;
198 let s_addr = src as usize;
199
200 if d_addr == s_addr || n == 0 {
201 return dst;
202 }
203
204 if d_addr < s_addr || d_addr.wrapping_sub(s_addr) >= n {
205 return unsafe { copy_nonoverlapping_bytes(dst, src, n) };
206 }
207
208 let dst = black_box(dst);
210 let src = black_box(src);
211 let mut i = n;
212
213 if i >= 16 {
214 let end_misalign = (d_addr.wrapping_add(i)) & 15;
215 if end_misalign != 0 {
216 let target = i - end_misalign;
217 while i > target {
218 i -= 1;
219 unsafe {
220 *dst.add(i) = black_box(*src.add(i));
221 }
222 }
223 }
224
225 let src_off = (s_addr.wrapping_add(i)) & 7;
226 if src_off == 0 {
227 while i >= 64 {
228 i -= 64;
229 unsafe {
230 let s = src.add(i).cast::<u64>();
231 let d = dst.add(i).cast::<u64>();
232 let w0 = black_box(*s);
233 let w1 = *s.add(1);
234 let w2 = *s.add(2);
235 let w3 = *s.add(3);
236 let w4 = *s.add(4);
237 let w5 = *s.add(5);
238 let w6 = *s.add(6);
239 let w7 = *s.add(7);
240 *d.add(7) = w7;
241 *d.add(6) = w6;
242 *d.add(5) = w5;
243 *d.add(4) = w4;
244 *d.add(3) = w3;
245 *d.add(2) = w2;
246 *d.add(1) = w1;
247 *d = w0;
248 }
249 }
250
251 if i >= 32 {
252 i -= 32;
253 unsafe {
254 let s = src.add(i);
255 let d = dst.add(i);
256 let w0 = read_u64_aligned(s);
257 let w1 = read_u64_aligned(s.add(8));
258 let w2 = read_u64_aligned(s.add(16));
259 let w3 = read_u64_aligned(s.add(24));
260 write_u64_aligned(d.add(24), w3);
261 write_u64_aligned(d.add(16), w2);
262 write_u64_aligned(d.add(8), w1);
263 write_u64_aligned(d, w0);
264 }
265 }
266
267 if i >= 16 {
268 i -= 16;
269 unsafe {
270 let s = src.add(i);
271 let d = dst.add(i);
272 let w0 = read_u64_aligned(s);
273 let w1 = read_u64_aligned(s.add(8));
274 write_u64_aligned(d.add(8), w1);
275 write_u64_aligned(d, w0);
276 }
277 }
278
279 if i >= 8 {
280 i -= 8;
281 unsafe {
282 let w = read_u64_aligned(src.add(i));
283 write_u64_aligned(dst.add(i), w);
284 }
285 }
286 } else if i >= 16 {
287 let k = 8 - src_off;
288 let rshift = (src_off * 8) as u32;
289 let lshift = (k * 8) as u32;
290 let mut high: u64 = 0;
291 let mut b = 0;
292 while b < src_off {
293 high |= (unsafe { *src.add(i - src_off + b) } as u64) << ((k + b) * 8);
294 b += 1;
295 }
296
297 while i >= 40 {
298 unsafe {
299 let s_al = src.add(i - 32 - src_off);
300 let d_al = dst.add(i - 32);
301 let w0 = u64::from_le(read_u64_aligned(s_al));
302 let w1 = u64::from_le(read_u64_aligned(s_al.add(8)));
303 let w2 = u64::from_le(read_u64_aligned(s_al.add(16)));
304 let w3 = u64::from_le(read_u64_aligned(s_al.add(24)));
305 write_u64_aligned(d_al.add(24), ((w3 >> rshift) | high).to_le());
306 write_u64_aligned(d_al.add(16), ((w2 >> rshift) | (w3 << lshift)).to_le());
307 write_u64_aligned(d_al.add(8), ((w1 >> rshift) | (w2 << lshift)).to_le());
308 write_u64_aligned(d_al, ((w0 >> rshift) | (w1 << lshift)).to_le());
309 high = w0 << lshift;
310 }
311 i -= 32;
312 }
313
314 while i >= 16 {
315 unsafe {
316 let w = u64::from_le(read_u64_aligned(src.add(i - 8 - src_off)));
317 write_u64_aligned(dst.add(i - 8), ((w >> rshift) | high).to_le());
318 high = w << lshift;
319 }
320 i -= 8;
321 }
322 }
323 }
324
325 while i > 0 {
326 i -= 1;
327 unsafe {
328 *dst.add(i) = black_box(*src.add(i));
329 }
330 }
331
332 dst
333}
334
335pub unsafe fn set_bytes(dst: *mut u8, byte: u8, n: usize) -> *mut u8 {
340 let dst = black_box(dst);
341 let mut i = 0;
342 let splat = black_box(LO_U64.wrapping_mul(byte as u64));
343
344 if n >= 16 {
345 let misalign = (dst as usize) & 15;
346 if misalign != 0 {
347 let prolog = (16 - misalign) & 15;
348 while i < prolog {
349 unsafe {
350 *dst.add(i) = black_box(byte);
351 }
352 i += 1;
353 }
354 }
355
356 while n - i >= 64 {
357 unsafe {
358 let d = dst.add(i);
359 write_u64_aligned(d, splat);
360 write_u64_aligned(d.add(8), splat);
361 write_u64_aligned(d.add(16), splat);
362 write_u64_aligned(d.add(24), splat);
363 write_u64_aligned(d.add(32), splat);
364 write_u64_aligned(d.add(40), splat);
365 write_u64_aligned(d.add(48), splat);
366 write_u64_aligned(d.add(56), splat);
367 }
368 i += 64;
369 }
370
371 if n - i >= 32 {
372 unsafe {
373 let d = dst.add(i);
374 write_u64_aligned(d, splat);
375 write_u64_aligned(d.add(8), splat);
376 write_u64_aligned(d.add(16), splat);
377 write_u64_aligned(d.add(24), splat);
378 }
379 i += 32;
380 }
381
382 if n - i >= 16 {
383 unsafe {
384 let d = dst.add(i);
385 write_u64_aligned(d, splat);
386 write_u64_aligned(d.add(8), splat);
387 }
388 i += 16;
389 }
390
391 if n - i >= 8 {
392 unsafe {
393 write_u64_aligned(dst.add(i), splat);
394 }
395 i += 8;
396 }
397 }
398
399 while i < n {
400 unsafe {
401 *dst.add(i) = black_box(byte);
402 }
403 i += 1;
404 }
405
406 dst
407}
408
409pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 {
414 let a = black_box(a);
415 let b = black_box(b);
416 let mut i = 0;
417
418 if n >= 16 {
419 while i < n && ((a as usize).wrapping_add(i) & 7) != 0 {
420 let av = unsafe { *a.add(i) };
421 let bv = unsafe { *b.add(i) };
422 if av != bv {
423 return (av as i32) - (bv as i32);
424 }
425 i += 1;
426 }
427
428 let b_off = ((b as usize).wrapping_add(i)) & 7;
429 if b_off == 0 {
430 while n - i >= 8 {
431 let wa = unsafe { read_u64_aligned(a.add(i)) };
432 let wb = unsafe { read_u64_aligned(b.add(i)) };
433 if wa != wb {
434 let diff = u64::from_be(wa) ^ u64::from_be(wb);
435 let idx = (diff.leading_zeros() >> 3) as usize;
436 let av = unsafe { *a.add(i + idx) };
437 let bv = unsafe { *b.add(i + idx) };
438 return (av as i32) - (bv as i32);
439 }
440 i += 8;
441 }
442 } else if n - i >= 16 {
443 let k = 8 - b_off;
444 let rshift = (b_off * 8) as u32;
445 let lshift = (k * 8) as u32;
446 let mut low: u64 = 0;
447 let mut j = 0;
448 while j < k {
449 low |= (unsafe { *b.add(i + j) } as u64) << (j * 8);
450 j += 1;
451 }
452 while n - i >= 16 {
453 let wa = unsafe { read_u64_aligned(a.add(i)) };
454 let wb_next = u64::from_le(unsafe { read_u64_aligned(b.add(i + k)) });
455 let wb = (low | (wb_next << lshift)).to_le();
456 if wa != wb {
457 let diff = u64::from_be(wa) ^ u64::from_be(wb);
458 let idx = (diff.leading_zeros() >> 3) as usize;
459 let av = unsafe { *a.add(i + idx) };
460 let bv = unsafe { *b.add(i + idx) };
461 return (av as i32) - (bv as i32);
462 }
463 low = wb_next >> rshift;
464 i += 8;
465 }
466 }
467 }
468
469 while i < n {
470 let av = unsafe { *a.add(i) };
471 let bv = unsafe { *b.add(i) };
472 if av != bv {
473 return (av as i32) - (bv as i32);
474 }
475 i += 1;
476 }
477 0
478}
479
480pub unsafe fn c_strlen(s: *const u8) -> usize {
485 let s = black_box(s);
486 let mut len = 0;
487
488 while ((s as usize).wrapping_add(len) & 7) != 0 {
489 if unsafe { *s.add(len) } == 0 {
490 return len;
491 }
492 len += 1;
493 }
494
495 loop {
496 let word = u64::from_le(unsafe { *s.add(len).cast::<u64>() });
497 let mask = has_zero_byte(word);
498 if mask != 0 {
499 return len + first_match_index_le(mask);
500 }
501 len += 8;
502 }
503}
504
505pub unsafe fn c_strnlen(s: *const u8, max_len: usize) -> usize {
510 let s = black_box(s);
511 let mut len = 0;
512
513 while len < max_len && ((s as usize).wrapping_add(len) & 7) != 0 {
514 if unsafe { *s.add(len) } == 0 {
515 return len;
516 }
517 len += 1;
518 }
519
520 while max_len - len >= 8 {
521 let word = u64::from_le(unsafe { *s.add(len).cast::<u64>() });
522 let mask = has_zero_byte(word);
523 if mask != 0 {
524 return len + first_match_index_le(mask);
525 }
526 len += 8;
527 }
528
529 while len < max_len {
530 if unsafe { *s.add(len) } == 0 {
531 return len;
532 }
533 len += 1;
534 }
535 len
536}
537
538pub fn memchr(needle: u8, haystack: &[u8]) -> Option<usize> {
540 let n = haystack.len();
541 let ptr = black_box(haystack.as_ptr());
542 let mut i = 0;
543 let splat = LO_U64.wrapping_mul(needle as u64);
544
545 while i < n && ((ptr as usize).wrapping_add(i) & 7) != 0 {
546 if haystack[i] == needle {
547 return Some(i);
548 }
549 i += 1;
550 }
551
552 while n - i >= 8 {
553 let word = u64::from_le(unsafe { *ptr.add(i).cast::<u64>() }) ^ splat;
554 let mask = has_zero_byte(word);
555 if mask != 0 {
556 return Some(i + first_match_index_le(mask));
557 }
558 i += 8;
559 }
560
561 while i < n {
562 if haystack[i] == needle {
563 return Some(i);
564 }
565 i += 1;
566 }
567 None
568}
569
570pub fn memrchr(needle: u8, haystack: &[u8]) -> Option<usize> {
572 let ptr = black_box(haystack.as_ptr());
573 let mut i = haystack.len();
574 let splat = LO_U64.wrapping_mul(needle as u64);
575
576 while i > 0 && ((ptr as usize).wrapping_add(i) & 7) != 0 {
577 i -= 1;
578 if haystack[i] == needle {
579 return Some(i);
580 }
581 }
582
583 while i >= 8 {
584 i -= 8;
585 let word = u64::from_le(unsafe { *ptr.add(i).cast::<u64>() }) ^ splat;
586 let mask = has_zero_byte(word);
587 if mask != 0 {
588 return Some(i + last_match_index_le(mask));
589 }
590 }
591
592 while i > 0 {
593 i -= 1;
594 if haystack[i] == needle {
595 return Some(i);
596 }
597 }
598 None
599}
600
601pub fn ct_eq(a: &[u8], b: &[u8]) -> bool {
603 if a.len() != b.len() {
604 return false;
605 }
606 let mut diff: u8 = 0;
607 let mut i = 0;
608 while i < a.len() {
609 diff |= black_box(a[i] ^ b[i]);
610 i += 1;
611 }
612 black_box(diff) == 0
613}
614
615#[cfg(all(feature = "symbols", target_os = "none"))]
621#[unsafe(no_mangle)]
622pub unsafe extern "C" fn memcpy(
623 dst: *mut core::ffi::c_void,
624 src: *const core::ffi::c_void,
625 n: usize,
626) -> *mut core::ffi::c_void {
627 unsafe { copy_nonoverlapping_bytes(dst.cast::<u8>(), src.cast::<u8>(), n).cast() }
628}
629
630#[cfg(all(feature = "symbols", target_os = "none"))]
635#[unsafe(no_mangle)]
636pub unsafe extern "C" fn memmove(
637 dst: *mut core::ffi::c_void,
638 src: *const core::ffi::c_void,
639 n: usize,
640) -> *mut core::ffi::c_void {
641 unsafe { copy_bytes(dst.cast::<u8>(), src.cast::<u8>(), n).cast() }
642}
643
644#[cfg(all(feature = "symbols", target_os = "none"))]
649#[unsafe(no_mangle)]
650pub unsafe extern "C" fn memset(
651 dst: *mut core::ffi::c_void,
652 c: core::ffi::c_int,
653 n: usize,
654) -> *mut core::ffi::c_void {
655 unsafe { set_bytes(dst.cast::<u8>(), c as u8, n).cast() }
656}
657
658#[cfg(all(feature = "symbols", target_os = "none"))]
663#[unsafe(no_mangle)]
664pub unsafe extern "C" fn memcmp(
665 a: *const core::ffi::c_void,
666 b: *const core::ffi::c_void,
667 n: usize,
668) -> core::ffi::c_int {
669 unsafe { compare_bytes(a.cast::<u8>(), b.cast::<u8>(), n) }
670}
671
672#[cfg(all(feature = "symbols", target_os = "none"))]
677#[unsafe(no_mangle)]
678pub unsafe extern "C" fn bcmp(
679 a: *const core::ffi::c_void,
680 b: *const core::ffi::c_void,
681 n: usize,
682) -> core::ffi::c_int {
683 unsafe { compare_bytes(a.cast::<u8>(), b.cast::<u8>(), n) }
684}
685
686#[cfg(all(feature = "symbols", target_os = "none"))]
691#[unsafe(no_mangle)]
692pub unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize {
693 unsafe { c_strlen(s.cast::<u8>()) }
694}
695
696#[cfg(all(feature = "symbols", target_os = "none"))]
701#[unsafe(no_mangle)]
702pub unsafe extern "C" fn strnlen(s: *const core::ffi::c_char, max_len: usize) -> usize {
703 unsafe { c_strnlen(s.cast::<u8>(), max_len) }
704}
705
706#[cfg(test)]
707mod tests {
708 use super::*;
709
710 #[test]
711 fn copy_aligned_and_unaligned() {
712 let mut src = [0u8; 64];
713 for (i, b) in src.iter_mut().enumerate() {
714 *b = (i as u8).wrapping_mul(3).wrapping_add(1);
715 }
716 let mut dst = [0u8; 64];
717
718 unsafe {
719 copy_nonoverlapping_bytes(dst.as_mut_ptr(), src.as_ptr(), 64);
720 }
721 assert_eq!(dst, src);
722
723 let mut unaligned_dst = [0u8; 64];
724 unsafe {
725 copy_nonoverlapping_bytes(unaligned_dst.as_mut_ptr().add(1), src.as_ptr().add(3), 29);
726 }
727 assert_eq!(&unaligned_dst[1..30], &src[3..32]);
728 }
729
730 #[test]
731 fn copy_overlapping_forward_and_backward() {
732 let mut buf = [1u8, 2, 3, 4, 5, 6, 7, 8];
733 unsafe {
734 copy_bytes(buf.as_mut_ptr().add(2), buf.as_ptr(), 5);
735 }
736 assert_eq!(buf, [1, 2, 1, 2, 3, 4, 5, 8]);
737
738 let mut buf2 = [1u8, 2, 3, 4, 5, 6, 7, 8];
739 unsafe {
740 copy_bytes(buf2.as_mut_ptr(), buf2.as_ptr().add(2), 5);
741 }
742 assert_eq!(buf2, [3, 4, 5, 6, 7, 6, 7, 8]);
743 }
744
745 #[test]
746 fn set_and_compare() {
747 let mut buf = [0u8; 31];
748 unsafe {
749 set_bytes(buf.as_mut_ptr(), 0x5a, 31);
750 }
751 assert!(buf.iter().all(|&b| b == 0x5a));
752
753 let other = [0x5au8; 31];
754 assert_eq!(
755 unsafe { compare_bytes(buf.as_ptr(), other.as_ptr(), 31) },
756 0
757 );
758
759 buf[17] = 0x5b;
760 assert!(unsafe { compare_bytes(buf.as_ptr(), other.as_ptr(), 31) } > 0);
761 assert!(unsafe { compare_bytes(other.as_ptr(), buf.as_ptr(), 31) } < 0);
762 }
763
764 #[test]
765 fn c_strings_and_slice_helpers() {
766 let s = b"hello\0world";
767 assert_eq!(unsafe { c_strlen(s.as_ptr()) }, 5);
768 assert_eq!(unsafe { c_strnlen(s.as_ptr(), 3) }, 3);
769 assert_eq!(unsafe { c_strnlen(s.as_ptr(), 16) }, 5);
770
771 assert_eq!(memchr(b'l', b"hello"), Some(2));
772 assert_eq!(memrchr(b'l', b"hello"), Some(3));
773 assert_eq!(memchr(b'z', b"hello"), None);
774
775 assert!(ct_eq(b"secret-key", b"secret-key"));
776 assert!(!ct_eq(b"secret-key", b"secret-kez"));
777 assert!(!ct_eq(b"short", b"longer"));
778 }
779
780 #[test]
781 fn bulk_misaligned_offsets_and_overlaps() {
782 let mut src = [0u8; 320];
783 for (i, b) in src.iter_mut().enumerate() {
784 *b = (i as u8).wrapping_mul(17).wrapping_add(5);
785 }
786
787 for src_off in 0..16 {
788 for dst_off in 0..16 {
789 for &len in &[0usize, 1, 7, 8, 15, 16, 31, 63, 64, 65, 127, 128, 193, 256] {
790 let mut dst = [0xeeu8; 320];
791 unsafe {
792 copy_nonoverlapping_bytes(
793 dst.as_mut_ptr().add(dst_off),
794 src.as_ptr().add(src_off),
795 len,
796 );
797 }
798 assert_eq!(&dst[..dst_off], &[0xeeu8; 16][..dst_off]);
799 assert_eq!(&dst[dst_off..dst_off + len], &src[src_off..src_off + len]);
800 assert_eq!(&dst[dst_off + len..dst_off + len + 8], &[0xeeu8; 8]);
801 }
802 }
803 }
804
805 for shift in 1..16 {
806 for base_off in 0..8 {
807 for &len in &[1usize, 7, 8, 15, 16, 31, 64, 97, 128, 192, 256] {
808 let mut buf = src;
809 unsafe {
810 copy_bytes(
811 buf.as_mut_ptr().add(base_off + shift),
812 buf.as_ptr().add(base_off),
813 len,
814 );
815 }
816 assert_eq!(
817 &buf[base_off + shift..base_off + shift + len],
818 &src[base_off..base_off + len]
819 );
820
821 let mut buf_fwd = src;
822 unsafe {
823 copy_bytes(
824 buf_fwd.as_mut_ptr().add(base_off),
825 buf_fwd.as_ptr().add(base_off + shift),
826 len,
827 );
828 }
829 assert_eq!(
830 &buf_fwd[base_off..base_off + len],
831 &src[base_off + shift..base_off + shift + len]
832 );
833 }
834 }
835 }
836 }
837
838 #[test]
839 fn swar_strlen_memchr_and_word_compare_edges() {
840 let mut buf = [b'x'; 160];
841 for off in 0..16 {
842 for len in 0..96 {
843 buf.fill(b'a');
844 if len >= 2 {
845 buf[off + len / 2] = 0x80;
846 buf[off + len - 1] = 0xff;
847 }
848 buf[off + len] = 0;
849
850 let ptr = unsafe { buf.as_ptr().add(off) };
851 assert_eq!(unsafe { c_strlen(ptr) }, len);
852 assert_eq!(unsafe { c_strnlen(ptr, len + 16) }, len);
853 if len > 0 {
854 assert_eq!(unsafe { c_strnlen(ptr, len - 1) }, len - 1);
855 }
856
857 let slice = &buf[off..off + len];
858 assert_eq!(memchr(0, &buf[off..off + len + 1]), Some(len));
859 assert_eq!(memchr(b'z', slice), None);
860 assert_eq!(memrchr(b'z', slice), None);
861 }
862 }
863
864 let mut hay = [b'.'; 128];
865 for off in 0..8 {
866 for pos in 0..80 {
867 hay.fill(b'.');
868 hay[off + pos] = 0xa5;
869 let s = &hay[off..off + 80];
870 assert_eq!(memchr(0xa5, s), Some(pos));
871 assert_eq!(memrchr(0xa5, s), Some(pos));
872
873 let mut other = [b'.'; 128];
874 other[off + pos] = 0xa4;
875 let a_ptr = unsafe { hay.as_ptr().add(off) };
876 assert!(unsafe { compare_bytes(a_ptr, other.as_ptr().add(off), 80) } > 0);
877 assert!(unsafe { compare_bytes(other.as_ptr().add(off), a_ptr, 80) } < 0);
878 other[off + pos] = 0xa5;
879 assert_eq!(
880 unsafe { compare_bytes(a_ptr, other.as_ptr().add(off), 80) },
881 0
882 );
883 }
884 }
885 }
886}