Skip to main content

libc_rs/
mem.rs

1//! Freestanding memory and byte-slice intrinsics.
2//!
3//! All raw primitives avoid calling `core::ptr::copy` or `write_bytes` and
4//! pass leading pointers through `core::hint::black_box` once at function entry
5//! so LLVM's `LoopIdiomRecognize` pass cannot lower them into recursive calls
6//! to `memcpy`, `memmove`, `memset`, `memcmp`, or `strlen` when the `symbols`
7//! feature is enabled.
8
9use core::hint::black_box;
10
11const LO_U64: u64 = 0x0101_0101_0101_0101;
12const HI_U64: u64 = 0x8080_8080_8080_8080;
13
14#[inline(always)]
15fn has_zero_byte(word: u64) -> u64 {
16    word.wrapping_sub(LO_U64) & !word & HI_U64
17}
18
19#[inline(always)]
20fn first_match_index_le(mask: u64) -> usize {
21    (mask.trailing_zeros() >> 3) as usize
22}
23
24#[inline(always)]
25fn last_match_index_le(mask: u64) -> usize {
26    7 - ((mask.leading_zeros() >> 3) as usize)
27}
28
29#[inline(always)]
30unsafe fn read_u64_aligned(ptr: *const u8) -> u64 {
31    debug_assert_eq!((ptr as usize) & 7, 0);
32    unsafe { *ptr.cast::<u64>() }
33}
34
35#[inline(always)]
36unsafe fn write_u64_aligned(ptr: *mut u8, val: u64) {
37    debug_assert_eq!((ptr as usize) & 7, 0);
38    unsafe {
39        *ptr.cast::<u64>() = val;
40    }
41}
42
43#[cold]
44#[inline(never)]
45unsafe fn copy_forward_misaligned(
46    dst: *mut u8,
47    src: *const u8,
48    mut i: usize,
49    n: usize,
50    src_off: usize,
51) -> usize {
52    if n - i >= 16 {
53        let k = 8 - src_off;
54        let rshift = (src_off * 8) as u32;
55        let lshift = (k * 8) as u32;
56        let mut low: u64 = 0;
57        let mut b = 0;
58        while b < k {
59            low |= (unsafe { *src.add(i + b) } as u64) << (b * 8);
60            b += 1;
61        }
62
63        while n - i >= 40 {
64            unsafe {
65                let s_al = src.add(i + k);
66                let d_al = dst.add(i);
67                let w0 = u64::from_le(read_u64_aligned(s_al));
68                let w1 = u64::from_le(read_u64_aligned(s_al.add(8)));
69                let w2 = u64::from_le(read_u64_aligned(s_al.add(16)));
70                let w3 = u64::from_le(read_u64_aligned(s_al.add(24)));
71                write_u64_aligned(d_al, (low | (w0 << lshift)).to_le());
72                write_u64_aligned(d_al.add(8), ((w0 >> rshift) | (w1 << lshift)).to_le());
73                write_u64_aligned(d_al.add(16), ((w1 >> rshift) | (w2 << lshift)).to_le());
74                write_u64_aligned(d_al.add(24), ((w2 >> rshift) | (w3 << lshift)).to_le());
75                low = w3 >> rshift;
76            }
77            i += 32;
78        }
79
80        while n - i >= 16 {
81            unsafe {
82                let w0 = u64::from_le(read_u64_aligned(src.add(i + k)));
83                write_u64_aligned(dst.add(i), (low | (w0 << lshift)).to_le());
84                low = w0 >> rshift;
85            }
86            i += 8;
87        }
88    }
89    i
90}
91
92/// Copies `n` bytes from `src` to `dst`.
93///
94/// # Safety
95/// - `src` must be valid for reads of `n` bytes.
96/// - `dst` must be valid for writes of `n` bytes.
97/// - The memory regions must not overlap.
98pub unsafe fn copy_nonoverlapping_bytes(dst: *mut u8, src: *const u8, n: usize) -> *mut u8 {
99    let dst = black_box(dst);
100    let src = black_box(src);
101    let mut i = 0;
102
103    if n >= 16 {
104        let misalign = (dst as usize) & 15;
105        if misalign != 0 {
106            let prolog = (16 - misalign) & 15;
107            while i < prolog {
108                unsafe {
109                    *dst.add(i) = black_box(*src.add(i));
110                }
111                i += 1;
112            }
113        }
114
115        let src_off = (unsafe { src.add(i) } as usize) & 7;
116        if src_off == 0 {
117            while n - i >= 64 {
118                unsafe {
119                    let s = src.add(i).cast::<u64>();
120                    let d = dst.add(i).cast::<u64>();
121                    let w0 = black_box(*s);
122                    let w1 = *s.add(1);
123                    let w2 = *s.add(2);
124                    let w3 = *s.add(3);
125                    let w4 = *s.add(4);
126                    let w5 = *s.add(5);
127                    let w6 = *s.add(6);
128                    let w7 = *s.add(7);
129                    *d = w0;
130                    *d.add(1) = w1;
131                    *d.add(2) = w2;
132                    *d.add(3) = w3;
133                    *d.add(4) = w4;
134                    *d.add(5) = w5;
135                    *d.add(6) = w6;
136                    *d.add(7) = w7;
137                }
138                i += 64;
139            }
140
141            if n - i >= 32 {
142                unsafe {
143                    let s = src.add(i);
144                    let d = dst.add(i);
145                    let w0 = read_u64_aligned(s);
146                    let w1 = read_u64_aligned(s.add(8));
147                    let w2 = read_u64_aligned(s.add(16));
148                    let w3 = read_u64_aligned(s.add(24));
149                    write_u64_aligned(d, w0);
150                    write_u64_aligned(d.add(8), w1);
151                    write_u64_aligned(d.add(16), w2);
152                    write_u64_aligned(d.add(24), w3);
153                }
154                i += 32;
155            }
156
157            if n - i >= 16 {
158                unsafe {
159                    let s = src.add(i);
160                    let d = dst.add(i);
161                    let w0 = read_u64_aligned(s);
162                    let w1 = read_u64_aligned(s.add(8));
163                    write_u64_aligned(d, w0);
164                    write_u64_aligned(d.add(8), w1);
165                }
166                i += 16;
167            }
168
169            if n - i >= 8 {
170                unsafe {
171                    let w = read_u64_aligned(src.add(i));
172                    write_u64_aligned(dst.add(i), w);
173                }
174                i += 8;
175            }
176        } else {
177            i = unsafe { copy_forward_misaligned(dst, src, i, n, src_off) };
178        }
179    }
180
181    while i < n {
182        unsafe {
183            *dst.add(i) = black_box(*src.add(i));
184        }
185        i += 1;
186    }
187
188    dst
189}
190
191/// Copies `n` bytes from `src` to `dst`. The regions may overlap.
192///
193/// # Safety
194/// - `src` must be valid for reads of `n` bytes.
195/// - `dst` must be valid for writes of `n` bytes.
196pub unsafe fn copy_bytes(dst: *mut u8, src: *const u8, n: usize) -> *mut u8 {
197    let d_addr = dst as usize;
198    let s_addr = src as usize;
199
200    if d_addr == s_addr || n == 0 {
201        return dst;
202    }
203
204    if d_addr < s_addr || d_addr.wrapping_sub(s_addr) >= n {
205        return unsafe { copy_nonoverlapping_bytes(dst, src, n) };
206    }
207
208    // Overlapping with dst > src: copy backwards from the end using strictly aligned word ops.
209    let dst = black_box(dst);
210    let src = black_box(src);
211    let mut i = n;
212
213    if i >= 16 {
214        let end_misalign = (d_addr.wrapping_add(i)) & 15;
215        if end_misalign != 0 {
216            let target = i - end_misalign;
217            while i > target {
218                i -= 1;
219                unsafe {
220                    *dst.add(i) = black_box(*src.add(i));
221                }
222            }
223        }
224
225        let src_off = (s_addr.wrapping_add(i)) & 7;
226        if src_off == 0 {
227            while i >= 64 {
228                i -= 64;
229                unsafe {
230                    let s = src.add(i).cast::<u64>();
231                    let d = dst.add(i).cast::<u64>();
232                    let w0 = black_box(*s);
233                    let w1 = *s.add(1);
234                    let w2 = *s.add(2);
235                    let w3 = *s.add(3);
236                    let w4 = *s.add(4);
237                    let w5 = *s.add(5);
238                    let w6 = *s.add(6);
239                    let w7 = *s.add(7);
240                    *d.add(7) = w7;
241                    *d.add(6) = w6;
242                    *d.add(5) = w5;
243                    *d.add(4) = w4;
244                    *d.add(3) = w3;
245                    *d.add(2) = w2;
246                    *d.add(1) = w1;
247                    *d = w0;
248                }
249            }
250
251            if i >= 32 {
252                i -= 32;
253                unsafe {
254                    let s = src.add(i);
255                    let d = dst.add(i);
256                    let w0 = read_u64_aligned(s);
257                    let w1 = read_u64_aligned(s.add(8));
258                    let w2 = read_u64_aligned(s.add(16));
259                    let w3 = read_u64_aligned(s.add(24));
260                    write_u64_aligned(d.add(24), w3);
261                    write_u64_aligned(d.add(16), w2);
262                    write_u64_aligned(d.add(8), w1);
263                    write_u64_aligned(d, w0);
264                }
265            }
266
267            if i >= 16 {
268                i -= 16;
269                unsafe {
270                    let s = src.add(i);
271                    let d = dst.add(i);
272                    let w0 = read_u64_aligned(s);
273                    let w1 = read_u64_aligned(s.add(8));
274                    write_u64_aligned(d.add(8), w1);
275                    write_u64_aligned(d, w0);
276                }
277            }
278
279            if i >= 8 {
280                i -= 8;
281                unsafe {
282                    let w = read_u64_aligned(src.add(i));
283                    write_u64_aligned(dst.add(i), w);
284                }
285            }
286        } else if i >= 16 {
287            let k = 8 - src_off;
288            let rshift = (src_off * 8) as u32;
289            let lshift = (k * 8) as u32;
290            let mut high: u64 = 0;
291            let mut b = 0;
292            while b < src_off {
293                high |= (unsafe { *src.add(i - src_off + b) } as u64) << ((k + b) * 8);
294                b += 1;
295            }
296
297            while i >= 40 {
298                unsafe {
299                    let s_al = src.add(i - 32 - src_off);
300                    let d_al = dst.add(i - 32);
301                    let w0 = u64::from_le(read_u64_aligned(s_al));
302                    let w1 = u64::from_le(read_u64_aligned(s_al.add(8)));
303                    let w2 = u64::from_le(read_u64_aligned(s_al.add(16)));
304                    let w3 = u64::from_le(read_u64_aligned(s_al.add(24)));
305                    write_u64_aligned(d_al.add(24), ((w3 >> rshift) | high).to_le());
306                    write_u64_aligned(d_al.add(16), ((w2 >> rshift) | (w3 << lshift)).to_le());
307                    write_u64_aligned(d_al.add(8), ((w1 >> rshift) | (w2 << lshift)).to_le());
308                    write_u64_aligned(d_al, ((w0 >> rshift) | (w1 << lshift)).to_le());
309                    high = w0 << lshift;
310                }
311                i -= 32;
312            }
313
314            while i >= 16 {
315                unsafe {
316                    let w = u64::from_le(read_u64_aligned(src.add(i - 8 - src_off)));
317                    write_u64_aligned(dst.add(i - 8), ((w >> rshift) | high).to_le());
318                    high = w << lshift;
319                }
320                i -= 8;
321            }
322        }
323    }
324
325    while i > 0 {
326        i -= 1;
327        unsafe {
328            *dst.add(i) = black_box(*src.add(i));
329        }
330    }
331
332    dst
333}
334
335/// Fills `n` bytes at `dst` with `byte`.
336///
337/// # Safety
338/// - `dst` must be valid for writes of `n` bytes.
339pub unsafe fn set_bytes(dst: *mut u8, byte: u8, n: usize) -> *mut u8 {
340    let dst = black_box(dst);
341    let mut i = 0;
342    let splat = black_box(LO_U64.wrapping_mul(byte as u64));
343
344    if n >= 16 {
345        let misalign = (dst as usize) & 15;
346        if misalign != 0 {
347            let prolog = (16 - misalign) & 15;
348            while i < prolog {
349                unsafe {
350                    *dst.add(i) = black_box(byte);
351                }
352                i += 1;
353            }
354        }
355
356        while n - i >= 64 {
357            unsafe {
358                let d = dst.add(i);
359                write_u64_aligned(d, splat);
360                write_u64_aligned(d.add(8), splat);
361                write_u64_aligned(d.add(16), splat);
362                write_u64_aligned(d.add(24), splat);
363                write_u64_aligned(d.add(32), splat);
364                write_u64_aligned(d.add(40), splat);
365                write_u64_aligned(d.add(48), splat);
366                write_u64_aligned(d.add(56), splat);
367            }
368            i += 64;
369        }
370
371        if n - i >= 32 {
372            unsafe {
373                let d = dst.add(i);
374                write_u64_aligned(d, splat);
375                write_u64_aligned(d.add(8), splat);
376                write_u64_aligned(d.add(16), splat);
377                write_u64_aligned(d.add(24), splat);
378            }
379            i += 32;
380        }
381
382        if n - i >= 16 {
383            unsafe {
384                let d = dst.add(i);
385                write_u64_aligned(d, splat);
386                write_u64_aligned(d.add(8), splat);
387            }
388            i += 16;
389        }
390
391        if n - i >= 8 {
392            unsafe {
393                write_u64_aligned(dst.add(i), splat);
394            }
395            i += 8;
396        }
397    }
398
399    while i < n {
400        unsafe {
401            *dst.add(i) = black_box(byte);
402        }
403        i += 1;
404    }
405
406    dst
407}
408
409/// Lexicographically compares `n` bytes at `a` and `b`.
410///
411/// # Safety
412/// - `a` and `b` must each be valid for reads of `n` bytes.
413pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 {
414    let a = black_box(a);
415    let b = black_box(b);
416    let mut i = 0;
417
418    if n >= 16 {
419        while i < n && ((a as usize).wrapping_add(i) & 7) != 0 {
420            let av = unsafe { *a.add(i) };
421            let bv = unsafe { *b.add(i) };
422            if av != bv {
423                return (av as i32) - (bv as i32);
424            }
425            i += 1;
426        }
427
428        let b_off = ((b as usize).wrapping_add(i)) & 7;
429        if b_off == 0 {
430            while n - i >= 8 {
431                let wa = unsafe { read_u64_aligned(a.add(i)) };
432                let wb = unsafe { read_u64_aligned(b.add(i)) };
433                if wa != wb {
434                    let diff = u64::from_be(wa) ^ u64::from_be(wb);
435                    let idx = (diff.leading_zeros() >> 3) as usize;
436                    let av = unsafe { *a.add(i + idx) };
437                    let bv = unsafe { *b.add(i + idx) };
438                    return (av as i32) - (bv as i32);
439                }
440                i += 8;
441            }
442        } else if n - i >= 16 {
443            let k = 8 - b_off;
444            let rshift = (b_off * 8) as u32;
445            let lshift = (k * 8) as u32;
446            let mut low: u64 = 0;
447            let mut j = 0;
448            while j < k {
449                low |= (unsafe { *b.add(i + j) } as u64) << (j * 8);
450                j += 1;
451            }
452            while n - i >= 16 {
453                let wa = unsafe { read_u64_aligned(a.add(i)) };
454                let wb_next = u64::from_le(unsafe { read_u64_aligned(b.add(i + k)) });
455                let wb = (low | (wb_next << lshift)).to_le();
456                if wa != wb {
457                    let diff = u64::from_be(wa) ^ u64::from_be(wb);
458                    let idx = (diff.leading_zeros() >> 3) as usize;
459                    let av = unsafe { *a.add(i + idx) };
460                    let bv = unsafe { *b.add(i + idx) };
461                    return (av as i32) - (bv as i32);
462                }
463                low = wb_next >> rshift;
464                i += 8;
465            }
466        }
467    }
468
469    while i < n {
470        let av = unsafe { *a.add(i) };
471        let bv = unsafe { *b.add(i) };
472        if av != bv {
473            return (av as i32) - (bv as i32);
474        }
475        i += 1;
476    }
477    0
478}
479
480/// Returns the length of a NUL-terminated byte string.
481///
482/// # Safety
483/// - `s` must point to a valid NUL-terminated byte string.
484pub unsafe fn c_strlen(s: *const u8) -> usize {
485    let s = black_box(s);
486    let mut len = 0;
487
488    while ((s as usize).wrapping_add(len) & 7) != 0 {
489        if unsafe { *s.add(len) } == 0 {
490            return len;
491        }
492        len += 1;
493    }
494
495    loop {
496        let word = u64::from_le(unsafe { *s.add(len).cast::<u64>() });
497        let mask = has_zero_byte(word);
498        if mask != 0 {
499            return len + first_match_index_le(mask);
500        }
501        len += 8;
502    }
503}
504
505/// Returns the length of a NUL-terminated byte string, bounded by `max_len`.
506///
507/// # Safety
508/// - `s` must be valid for reads up to `max_len` bytes or the first NUL byte.
509pub unsafe fn c_strnlen(s: *const u8, max_len: usize) -> usize {
510    let s = black_box(s);
511    let mut len = 0;
512
513    while len < max_len && ((s as usize).wrapping_add(len) & 7) != 0 {
514        if unsafe { *s.add(len) } == 0 {
515            return len;
516        }
517        len += 1;
518    }
519
520    while max_len - len >= 8 {
521        let word = u64::from_le(unsafe { *s.add(len).cast::<u64>() });
522        let mask = has_zero_byte(word);
523        if mask != 0 {
524            return len + first_match_index_le(mask);
525        }
526        len += 8;
527    }
528
529    while len < max_len {
530        if unsafe { *s.add(len) } == 0 {
531            return len;
532        }
533        len += 1;
534    }
535    len
536}
537
538/// Searches `haystack` from the front for the first occurrence of `needle`.
539pub fn memchr(needle: u8, haystack: &[u8]) -> Option<usize> {
540    let n = haystack.len();
541    let ptr = black_box(haystack.as_ptr());
542    let mut i = 0;
543    let splat = LO_U64.wrapping_mul(needle as u64);
544
545    while i < n && ((ptr as usize).wrapping_add(i) & 7) != 0 {
546        if haystack[i] == needle {
547            return Some(i);
548        }
549        i += 1;
550    }
551
552    while n - i >= 8 {
553        let word = u64::from_le(unsafe { *ptr.add(i).cast::<u64>() }) ^ splat;
554        let mask = has_zero_byte(word);
555        if mask != 0 {
556            return Some(i + first_match_index_le(mask));
557        }
558        i += 8;
559    }
560
561    while i < n {
562        if haystack[i] == needle {
563            return Some(i);
564        }
565        i += 1;
566    }
567    None
568}
569
570/// Searches `haystack` from the back for the last occurrence of `needle`.
571pub fn memrchr(needle: u8, haystack: &[u8]) -> Option<usize> {
572    let ptr = black_box(haystack.as_ptr());
573    let mut i = haystack.len();
574    let splat = LO_U64.wrapping_mul(needle as u64);
575
576    while i > 0 && ((ptr as usize).wrapping_add(i) & 7) != 0 {
577        i -= 1;
578        if haystack[i] == needle {
579            return Some(i);
580        }
581    }
582
583    while i >= 8 {
584        i -= 8;
585        let word = u64::from_le(unsafe { *ptr.add(i).cast::<u64>() }) ^ splat;
586        let mask = has_zero_byte(word);
587        if mask != 0 {
588            return Some(i + last_match_index_le(mask));
589        }
590    }
591
592    while i > 0 {
593        i -= 1;
594        if haystack[i] == needle {
595            return Some(i);
596        }
597    }
598    None
599}
600
601/// Constant-time byte-slice equality comparison.
602pub fn ct_eq(a: &[u8], b: &[u8]) -> bool {
603    if a.len() != b.len() {
604        return false;
605    }
606    let mut diff: u8 = 0;
607    let mut i = 0;
608    while i < a.len() {
609        diff |= black_box(a[i] ^ b[i]);
610        i += 1;
611    }
612    black_box(diff) == 0
613}
614
615/// Copies `n` non-overlapping bytes from `src` to `dst` and returns `dst`.
616///
617/// # Safety
618/// - `src` must be valid for reads of `n` bytes, and `dst` for writes of `n` bytes.
619/// - The memory regions must not overlap.
620#[cfg(all(feature = "symbols", target_os = "none"))]
621#[unsafe(no_mangle)]
622pub unsafe extern "C" fn memcpy(
623    dst: *mut core::ffi::c_void,
624    src: *const core::ffi::c_void,
625    n: usize,
626) -> *mut core::ffi::c_void {
627    unsafe { copy_nonoverlapping_bytes(dst.cast::<u8>(), src.cast::<u8>(), n).cast() }
628}
629
630/// Copies `n` potentially overlapping bytes from `src` to `dst` and returns `dst`.
631///
632/// # Safety
633/// - `src` must be valid for reads of `n` bytes, and `dst` for writes of `n` bytes.
634#[cfg(all(feature = "symbols", target_os = "none"))]
635#[unsafe(no_mangle)]
636pub unsafe extern "C" fn memmove(
637    dst: *mut core::ffi::c_void,
638    src: *const core::ffi::c_void,
639    n: usize,
640) -> *mut core::ffi::c_void {
641    unsafe { copy_bytes(dst.cast::<u8>(), src.cast::<u8>(), n).cast() }
642}
643
644/// Fills `n` bytes at `dst` with `c as u8` and returns `dst`.
645///
646/// # Safety
647/// - `dst` must be valid for writes of `n` bytes.
648#[cfg(all(feature = "symbols", target_os = "none"))]
649#[unsafe(no_mangle)]
650pub unsafe extern "C" fn memset(
651    dst: *mut core::ffi::c_void,
652    c: core::ffi::c_int,
653    n: usize,
654) -> *mut core::ffi::c_void {
655    unsafe { set_bytes(dst.cast::<u8>(), c as u8, n).cast() }
656}
657
658/// Lexicographically compares `n` bytes at `a` and `b`.
659///
660/// # Safety
661/// - `a` and `b` must each be valid for reads of `n` bytes.
662#[cfg(all(feature = "symbols", target_os = "none"))]
663#[unsafe(no_mangle)]
664pub unsafe extern "C" fn memcmp(
665    a: *const core::ffi::c_void,
666    b: *const core::ffi::c_void,
667    n: usize,
668) -> core::ffi::c_int {
669    unsafe { compare_bytes(a.cast::<u8>(), b.cast::<u8>(), n) }
670}
671
672/// Compares `n` bytes at `a` and `b` for equality (`0` when equal).
673///
674/// # Safety
675/// - `a` and `b` must each be valid for reads of `n` bytes.
676#[cfg(all(feature = "symbols", target_os = "none"))]
677#[unsafe(no_mangle)]
678pub unsafe extern "C" fn bcmp(
679    a: *const core::ffi::c_void,
680    b: *const core::ffi::c_void,
681    n: usize,
682) -> core::ffi::c_int {
683    unsafe { compare_bytes(a.cast::<u8>(), b.cast::<u8>(), n) }
684}
685
686/// Returns the length of a NUL-terminated byte string `s`.
687///
688/// # Safety
689/// - `s` must point to a valid NUL-terminated string.
690#[cfg(all(feature = "symbols", target_os = "none"))]
691#[unsafe(no_mangle)]
692pub unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize {
693    unsafe { c_strlen(s.cast::<u8>()) }
694}
695
696/// Returns the length of a NUL-terminated byte string `s`, bounded by `max_len`.
697///
698/// # Safety
699/// - `s` must be valid for reads up to `max_len` bytes or the first NUL byte.
700#[cfg(all(feature = "symbols", target_os = "none"))]
701#[unsafe(no_mangle)]
702pub unsafe extern "C" fn strnlen(s: *const core::ffi::c_char, max_len: usize) -> usize {
703    unsafe { c_strnlen(s.cast::<u8>(), max_len) }
704}
705
706#[cfg(test)]
707mod tests {
708    use super::*;
709
710    #[test]
711    fn copy_aligned_and_unaligned() {
712        let mut src = [0u8; 64];
713        for (i, b) in src.iter_mut().enumerate() {
714            *b = (i as u8).wrapping_mul(3).wrapping_add(1);
715        }
716        let mut dst = [0u8; 64];
717
718        unsafe {
719            copy_nonoverlapping_bytes(dst.as_mut_ptr(), src.as_ptr(), 64);
720        }
721        assert_eq!(dst, src);
722
723        let mut unaligned_dst = [0u8; 64];
724        unsafe {
725            copy_nonoverlapping_bytes(unaligned_dst.as_mut_ptr().add(1), src.as_ptr().add(3), 29);
726        }
727        assert_eq!(&unaligned_dst[1..30], &src[3..32]);
728    }
729
730    #[test]
731    fn copy_overlapping_forward_and_backward() {
732        let mut buf = [1u8, 2, 3, 4, 5, 6, 7, 8];
733        unsafe {
734            copy_bytes(buf.as_mut_ptr().add(2), buf.as_ptr(), 5);
735        }
736        assert_eq!(buf, [1, 2, 1, 2, 3, 4, 5, 8]);
737
738        let mut buf2 = [1u8, 2, 3, 4, 5, 6, 7, 8];
739        unsafe {
740            copy_bytes(buf2.as_mut_ptr(), buf2.as_ptr().add(2), 5);
741        }
742        assert_eq!(buf2, [3, 4, 5, 6, 7, 6, 7, 8]);
743    }
744
745    #[test]
746    fn set_and_compare() {
747        let mut buf = [0u8; 31];
748        unsafe {
749            set_bytes(buf.as_mut_ptr(), 0x5a, 31);
750        }
751        assert!(buf.iter().all(|&b| b == 0x5a));
752
753        let other = [0x5au8; 31];
754        assert_eq!(
755            unsafe { compare_bytes(buf.as_ptr(), other.as_ptr(), 31) },
756            0
757        );
758
759        buf[17] = 0x5b;
760        assert!(unsafe { compare_bytes(buf.as_ptr(), other.as_ptr(), 31) } > 0);
761        assert!(unsafe { compare_bytes(other.as_ptr(), buf.as_ptr(), 31) } < 0);
762    }
763
764    #[test]
765    fn c_strings_and_slice_helpers() {
766        let s = b"hello\0world";
767        assert_eq!(unsafe { c_strlen(s.as_ptr()) }, 5);
768        assert_eq!(unsafe { c_strnlen(s.as_ptr(), 3) }, 3);
769        assert_eq!(unsafe { c_strnlen(s.as_ptr(), 16) }, 5);
770
771        assert_eq!(memchr(b'l', b"hello"), Some(2));
772        assert_eq!(memrchr(b'l', b"hello"), Some(3));
773        assert_eq!(memchr(b'z', b"hello"), None);
774
775        assert!(ct_eq(b"secret-key", b"secret-key"));
776        assert!(!ct_eq(b"secret-key", b"secret-kez"));
777        assert!(!ct_eq(b"short", b"longer"));
778    }
779
780    #[test]
781    fn bulk_misaligned_offsets_and_overlaps() {
782        let mut src = [0u8; 320];
783        for (i, b) in src.iter_mut().enumerate() {
784            *b = (i as u8).wrapping_mul(17).wrapping_add(5);
785        }
786
787        for src_off in 0..16 {
788            for dst_off in 0..16 {
789                for &len in &[0usize, 1, 7, 8, 15, 16, 31, 63, 64, 65, 127, 128, 193, 256] {
790                    let mut dst = [0xeeu8; 320];
791                    unsafe {
792                        copy_nonoverlapping_bytes(
793                            dst.as_mut_ptr().add(dst_off),
794                            src.as_ptr().add(src_off),
795                            len,
796                        );
797                    }
798                    assert_eq!(&dst[..dst_off], &[0xeeu8; 16][..dst_off]);
799                    assert_eq!(&dst[dst_off..dst_off + len], &src[src_off..src_off + len]);
800                    assert_eq!(&dst[dst_off + len..dst_off + len + 8], &[0xeeu8; 8]);
801                }
802            }
803        }
804
805        for shift in 1..16 {
806            for base_off in 0..8 {
807                for &len in &[1usize, 7, 8, 15, 16, 31, 64, 97, 128, 192, 256] {
808                    let mut buf = src;
809                    unsafe {
810                        copy_bytes(
811                            buf.as_mut_ptr().add(base_off + shift),
812                            buf.as_ptr().add(base_off),
813                            len,
814                        );
815                    }
816                    assert_eq!(
817                        &buf[base_off + shift..base_off + shift + len],
818                        &src[base_off..base_off + len]
819                    );
820
821                    let mut buf_fwd = src;
822                    unsafe {
823                        copy_bytes(
824                            buf_fwd.as_mut_ptr().add(base_off),
825                            buf_fwd.as_ptr().add(base_off + shift),
826                            len,
827                        );
828                    }
829                    assert_eq!(
830                        &buf_fwd[base_off..base_off + len],
831                        &src[base_off + shift..base_off + shift + len]
832                    );
833                }
834            }
835        }
836    }
837
838    #[test]
839    fn swar_strlen_memchr_and_word_compare_edges() {
840        let mut buf = [b'x'; 160];
841        for off in 0..16 {
842            for len in 0..96 {
843                buf.fill(b'a');
844                if len >= 2 {
845                    buf[off + len / 2] = 0x80;
846                    buf[off + len - 1] = 0xff;
847                }
848                buf[off + len] = 0;
849
850                let ptr = unsafe { buf.as_ptr().add(off) };
851                assert_eq!(unsafe { c_strlen(ptr) }, len);
852                assert_eq!(unsafe { c_strnlen(ptr, len + 16) }, len);
853                if len > 0 {
854                    assert_eq!(unsafe { c_strnlen(ptr, len - 1) }, len - 1);
855                }
856
857                let slice = &buf[off..off + len];
858                assert_eq!(memchr(0, &buf[off..off + len + 1]), Some(len));
859                assert_eq!(memchr(b'z', slice), None);
860                assert_eq!(memrchr(b'z', slice), None);
861            }
862        }
863
864        let mut hay = [b'.'; 128];
865        for off in 0..8 {
866            for pos in 0..80 {
867                hay.fill(b'.');
868                hay[off + pos] = 0xa5;
869                let s = &hay[off..off + 80];
870                assert_eq!(memchr(0xa5, s), Some(pos));
871                assert_eq!(memrchr(0xa5, s), Some(pos));
872
873                let mut other = [b'.'; 128];
874                other[off + pos] = 0xa4;
875                let a_ptr = unsafe { hay.as_ptr().add(off) };
876                assert!(unsafe { compare_bytes(a_ptr, other.as_ptr().add(off), 80) } > 0);
877                assert!(unsafe { compare_bytes(other.as_ptr().add(off), a_ptr, 80) } < 0);
878                other[off + pos] = 0xa5;
879                assert_eq!(
880                    unsafe { compare_bytes(a_ptr, other.as_ptr().add(off), 80) },
881                    0
882                );
883            }
884        }
885    }
886}