fory_core/util/
string_util.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::ptr;
19
20const MAX_HASH32: u64 = (1 << 31) - 1;
21
22#[allow(dead_code)]
23pub enum StringFlag {
24    LATIN1 = 0,
25    UTF8 = 1,
26}
27
28/// Swaps the high 8 bits and the low 8 bits of a 16-bit value.
29fn swap_endian(value: u16) -> u16 {
30    value.rotate_right(8)
31}
32
33/// Converts UTF-16 encoded data to UTF-8.
34pub fn to_utf8(utf16: &[u16], is_little_endian: bool) -> Result<Vec<u8>, String> {
35    // Pre-allocating capacity to avoid dynamic resizing.
36    // Longest case: 1 u16 to 3 u8.
37    let mut utf8_bytes: Vec<u8> = Vec::with_capacity(utf16.len() * 3);
38    let ptr = utf8_bytes.as_mut_ptr();
39    let mut offset = 0;
40    let mut iter = utf16.iter();
41    while let Some(&wc) = iter.next() {
42        let wc = if is_little_endian {
43            swap_endian(wc)
44        } else {
45            wc
46        };
47        match wc {
48            code_point if code_point < 0x80 => {
49                unsafe {
50                    ptr.add(offset).write(code_point as u8);
51                }
52                offset += 1;
53            }
54            code_point if code_point < 0x800 => {
55                let bytes = [
56                    ((code_point >> 6) & 0b1_1111) as u8 | 0b1100_0000,
57                    (code_point & 0b11_1111) as u8 | 0b1000_0000,
58                ];
59                unsafe {
60                    ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), 2);
61                }
62                offset += 2;
63            }
64            wc1 if (0xd800..=0xdbff).contains(&wc1) => {
65                if let Some(&wc2) = iter.next() {
66                    let wc2 = if is_little_endian {
67                        swap_endian(wc2)
68                    } else {
69                        wc2
70                    };
71                    if !(0xdc00..=0xdfff).contains(&wc2) {
72                        return Err("Invalid UTF-16 string: wrong surrogate pair".to_string());
73                    }
74                    let code_point =
75                        ((((wc1 as u32) - 0xd800) << 10) | ((wc2 as u32) - 0xdc00)) + 0x10000;
76                    let bytes = [
77                        ((code_point >> 18) & 0b111) as u8 | 0b1111_0000,
78                        ((code_point >> 12) & 0b11_1111) as u8 | 0b1000_0000,
79                        ((code_point >> 6) & 0b11_1111) as u8 | 0b1000_0000,
80                        (code_point & 0b11_1111) as u8 | 0b1000_0000,
81                    ];
82                    unsafe {
83                        ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), 4);
84                    }
85                    offset += 4;
86                } else {
87                    return Err("Invalid UTF-16 string: missing surrogate pair".to_string());
88                }
89            }
90            _ => {
91                let bytes = [
92                    ((wc >> 12) | 0b1110_0000) as u8,
93                    ((wc >> 6) & 0b11_1111) as u8 | 0b1000_0000,
94                    (wc & 0b11_1111) as u8 | 0b1000_0000,
95                ];
96                unsafe {
97                    ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), 3);
98                }
99                offset += 3;
100            }
101        }
102    }
103    unsafe {
104        utf8_bytes.set_len(offset);
105    }
106    Ok(utf8_bytes)
107}
108
109/// Converts a camelCase or PascalCase string to snake_case.
110pub fn to_snake_case(name: &str) -> String {
111    let mut result = String::with_capacity(name.len() + 4);
112    let chars: Vec<char> = name.chars().collect();
113
114    for (i, &c) in chars.iter().enumerate() {
115        if c.is_ascii_uppercase() {
116            if i > 0 {
117                let prev_upper = chars.get(i - 1).is_some_and(|c| c.is_ascii_uppercase());
118                let next_upper_or_end = chars.get(i + 1).map_or(true, |c| c.is_ascii_uppercase());
119                if !prev_upper || !next_upper_or_end {
120                    result.push('_');
121                }
122            }
123            result.push(c.to_ascii_lowercase());
124        } else {
125            result.push(c);
126        }
127    }
128    result
129}
130
131/// Converts a snake_case string to lowerCamelCase.
132pub fn to_camel_case(name: &str) -> String {
133    let mut result = String::with_capacity(name.len());
134    let mut capitalize_next = false;
135
136    for c in name.chars() {
137        if c == '_' {
138            capitalize_next = true;
139        } else if capitalize_next {
140            result.push(c.to_ascii_uppercase());
141            capitalize_next = false;
142        } else {
143            result.push(c);
144        }
145    }
146    result
147}
148
149#[allow(dead_code)]
150pub fn compute_string_hash(s: &str) -> u32 {
151    let mut hash: u64 = 17;
152    s.as_bytes().iter().for_each(|b| {
153        hash = (hash * 31) + (*b as u64);
154        while hash >= MAX_HASH32 {
155            hash /= 7;
156        }
157    });
158    hash as u32
159}
160
161#[cfg(target_feature = "neon")]
162use std::arch::aarch64::*;
163
164#[cfg(target_feature = "avx2")]
165use std::arch::x86_64::*;
166
167#[cfg(target_feature = "sse2")]
168use std::arch::x86_64::*;
169
170#[cfg(target_arch = "x86_64")]
171pub const MIN_DIM_SIZE_AVX: usize = 32;
172
173#[cfg(any(
174    target_arch = "x86",
175    target_arch = "x86_64",
176    all(target_arch = "aarch64", target_feature = "neon")
177))]
178pub const MIN_DIM_SIZE_SIMD: usize = 16;
179
180#[cfg(target_arch = "x86_64")]
181unsafe fn is_latin_avx(s: &str) -> bool {
182    let bytes = s.as_bytes();
183    let len = bytes.len();
184    let mut i = 0;
185    // SIMD skip ASCII
186    while i + MIN_DIM_SIZE_AVX <= len {
187        let chunk = _mm256_loadu_si256(bytes.as_ptr().add(i) as *const __m256i);
188        let hi_mask = _mm256_set1_epi8(0x80u8 as i8);
189        let masked = _mm256_and_si256(chunk, hi_mask);
190        let cmp = _mm256_cmpeq_epi8(masked, _mm256_setzero_si256());
191        if _mm256_movemask_epi8(cmp) != -1 {
192            break;
193        }
194        i += MIN_DIM_SIZE_AVX;
195    }
196    // check latin in remaining chars
197    let s_tail = &s[i..];
198    for c in s_tail.chars() {
199        if c as u32 > 0xFF {
200            return false;
201        }
202    }
203    true
204}
205
206#[cfg(target_feature = "sse2")]
207unsafe fn is_latin_sse(s: &str) -> bool {
208    let bytes = s.as_bytes();
209    let len = bytes.len();
210    let mut i = 0;
211    // SIMD skip ASCII
212    while i + MIN_DIM_SIZE_SIMD <= len {
213        let chunk = _mm_loadu_si128(bytes.as_ptr().add(i) as *const __m128i);
214        let hi_mask = _mm_set1_epi8(0x80u8 as i8);
215        let masked = _mm_and_si128(chunk, hi_mask);
216        let cmp = _mm_cmpeq_epi8(masked, _mm_setzero_si128());
217        if _mm_movemask_epi8(cmp) != 0xFFFF {
218            break;
219        }
220        i += MIN_DIM_SIZE_SIMD;
221    }
222    // check latin in remaining chars
223    let s_tail = &s[i..];
224    for c in s_tail.chars() {
225        if c as u32 > 0xFF {
226            return false;
227        }
228    }
229    true
230}
231
232#[cfg(target_feature = "neon")]
233unsafe fn is_latin_neon(s: &str) -> bool {
234    let bytes = s.as_bytes();
235    let len = bytes.len();
236    let mut i = 0;
237    // SIMD skip ASCII
238    while i + MIN_DIM_SIZE_SIMD <= len {
239        let chunk = vld1q_u8(bytes.as_ptr().add(i));
240        let hi_mask = vdupq_n_u8(0x80);
241        let masked = vandq_u8(chunk, hi_mask);
242        if vmaxvq_u8(masked) != 0 {
243            break;
244        }
245        i += MIN_DIM_SIZE_SIMD;
246    }
247    // check latin in remaining chars
248    let s_tail = &s[i..];
249    for c in s_tail.chars() {
250        if c as u32 > 0xFF {
251            return false;
252        }
253    }
254    true
255}
256
257fn is_latin_standard(s: &str) -> bool {
258    s.chars().all(|c| c as u32 <= 0xFF)
259}
260
261pub fn is_latin(s: &str) -> bool {
262    #[cfg(target_arch = "x86_64")]
263    {
264        if is_x86_feature_detected!("avx")
265            && is_x86_feature_detected!("fma")
266            && s.len() >= MIN_DIM_SIZE_AVX
267        {
268            return unsafe { is_latin_avx(s) };
269        }
270    }
271
272    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
273    {
274        if is_x86_feature_detected!("sse") && s.len() >= MIN_DIM_SIZE_SIMD {
275            return unsafe { is_latin_sse(s) };
276        }
277    }
278
279    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
280    {
281        if std::arch::is_aarch64_feature_detected!("neon") && s.len() >= MIN_DIM_SIZE_SIMD {
282            return unsafe { is_latin_neon(s) };
283        }
284    }
285    is_latin_standard(s)
286}
287
288#[cfg(target_arch = "x86_64")]
289unsafe fn get_latin1_length_avx(s: &str) -> i32 {
290    let bytes = s.as_bytes();
291    let len = bytes.len();
292    let mut count = 0;
293    // SIMD skip ASCII
294    while count + MIN_DIM_SIZE_AVX <= len {
295        let chunk = _mm256_loadu_si256(bytes.as_ptr().add(count) as *const __m256i);
296        let hi_mask = _mm256_set1_epi8(0x80u8 as i8);
297        let masked = _mm256_and_si256(chunk, hi_mask);
298        let cmp = _mm256_cmpeq_epi8(masked, _mm256_setzero_si256());
299        if _mm256_movemask_epi8(cmp) != -1 {
300            break;
301        }
302        count += MIN_DIM_SIZE_AVX;
303    }
304    // check latin in remaining chars
305    let s_tail = &s[count..];
306    for c in s_tail.chars() {
307        if c as u32 > 0xFF {
308            return -1;
309        }
310        count += 1;
311    }
312    count as i32
313}
314
315#[cfg(target_feature = "sse2")]
316unsafe fn get_latin1_length_sse(s: &str) -> i32 {
317    let bytes = s.as_bytes();
318    let len = bytes.len();
319    let mut count = 0;
320    // SIMD skip ASCII
321    while count + MIN_DIM_SIZE_SIMD <= len {
322        let chunk = _mm_loadu_si128(bytes.as_ptr().add(count) as *const __m128i);
323        let hi_mask = _mm_set1_epi8(0x80u8 as i8);
324        let masked = _mm_and_si128(chunk, hi_mask);
325        let cmp = _mm_cmpeq_epi8(masked, _mm_setzero_si128());
326        if _mm_movemask_epi8(cmp) != 0xFFFF {
327            break;
328        }
329        count += MIN_DIM_SIZE_SIMD;
330    }
331    // check latin in remaining chars
332    let s_tail = &s[count..];
333    for c in s_tail.chars() {
334        if c as u32 > 0xFF {
335            return -1;
336        }
337        count += 1;
338    }
339    count as i32
340}
341
342#[cfg(target_feature = "neon")]
343unsafe fn get_latin1_length_neon(s: &str) -> i32 {
344    let bytes = s.as_bytes();
345    let len = bytes.len();
346    let mut count = 0;
347    // SIMD skip ASCII
348    while count + MIN_DIM_SIZE_SIMD <= len {
349        let chunk = vld1q_u8(bytes.as_ptr().add(count));
350        let hi_mask = vdupq_n_u8(0x80);
351        let masked = vandq_u8(chunk, hi_mask);
352        if vmaxvq_u8(masked) != 0 {
353            break;
354        }
355        count += MIN_DIM_SIZE_SIMD;
356    }
357    // check latin in remaining chars
358    let s_tail = &s[count..];
359    for c in s_tail.chars() {
360        if c as u32 > 0xFF {
361            return -1;
362        }
363        count += 1;
364    }
365    count as i32
366}
367
368fn get_latin1_length_standard(s: &str) -> i32 {
369    let mut count = 0;
370    for c in s.chars() {
371        if c as u32 > 0xFF {
372            return -1;
373        }
374        count += 1;
375    }
376    count
377}
378
379pub fn get_latin1_length(s: &str) -> i32 {
380    #[cfg(target_arch = "x86_64")]
381    {
382        if is_x86_feature_detected!("avx")
383            && is_x86_feature_detected!("fma")
384            && s.len() >= MIN_DIM_SIZE_AVX
385        {
386            return unsafe { get_latin1_length_avx(s) };
387        }
388    }
389
390    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
391    {
392        if is_x86_feature_detected!("sse") && s.len() >= MIN_DIM_SIZE_SIMD {
393            return unsafe { get_latin1_length_sse(s) };
394        }
395    }
396
397    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
398    {
399        if std::arch::is_aarch64_feature_detected!("neon") && s.len() >= MIN_DIM_SIZE_SIMD {
400            return unsafe { get_latin1_length_neon(s) };
401        }
402    }
403    get_latin1_length_standard(s)
404}
405
406#[cfg(test)]
407mod latin_tests {
408    // Import content from external modules
409    use super::*;
410    use rand::Rng;
411
412    fn generate_random_string(length: usize) -> String {
413        const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
414        let mut rng = rand::thread_rng();
415
416        let result: String = (0..length)
417            .map(|_| {
418                let idx = rng.gen_range(0..CHARSET.len());
419                CHARSET[idx] as char
420            })
421            .collect();
422
423        result
424    }
425
426    #[test]
427    fn test_is_latin() {
428        let s = generate_random_string(1000);
429        let not_latin_str = generate_random_string(1000) + "abc\u{1234}";
430
431        #[cfg(target_arch = "x86_64")]
432        {
433            if is_x86_feature_detected!("avx") && is_x86_feature_detected!("fma") {
434                assert!(unsafe { is_latin_avx(&s) });
435                assert!(!unsafe { is_latin_avx(&not_latin_str) });
436            }
437        }
438
439        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
440        {
441            if is_x86_feature_detected!("sse") && s.len() >= MIN_DIM_SIZE_SIMD {
442                assert!(unsafe { is_latin_sse(&s) });
443                assert!(!unsafe { is_latin_sse(&not_latin_str) });
444            }
445        }
446
447        #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
448        {
449            if std::arch::is_aarch64_feature_detected!("neon") && s.len() >= MIN_DIM_SIZE_SIMD {
450                assert!(unsafe { is_latin_neon(&s) });
451                assert!(!unsafe { is_latin_neon(&not_latin_str) });
452            }
453        }
454        assert!(is_latin_standard(&s));
455        assert!(!is_latin_standard(&not_latin_str));
456    }
457}
458
459fn fmix64(mut k: u64) -> u64 {
460    k ^= k >> 33;
461    k = k.wrapping_mul(0xff51afd7ed558ccdu64);
462    k ^= k >> 33;
463    k = k.wrapping_mul(0xc4ceb9fe1a85ec53u64);
464    k ^= k >> 33;
465
466    k
467}
468
469pub fn murmurhash3_x64_128(bytes: &[u8], seed: u64) -> (u64, u64) {
470    let c1 = 0x87c37b91114253d5u64;
471    let c2 = 0x4cf5ad432745937fu64;
472    let read_size = 16;
473    let len = bytes.len() as u64;
474    let block_count = len / read_size;
475
476    let (mut h1, mut h2) = (seed, seed);
477
478    for i in 0..block_count as usize {
479        let offset = i * read_size as usize;
480        // Input byte slices can be arbitrarily aligned. Read MurmurHash3 blocks
481        // as explicit little-endian bytes instead of forming an aligned u64 slice.
482        let mut k1 = u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap());
483        let mut k2 = u64::from_le_bytes(bytes[offset + 8..offset + 16].try_into().unwrap());
484
485        k1 = k1.wrapping_mul(c1);
486        k1 = k1.rotate_left(31);
487        k1 = k1.wrapping_mul(c2);
488        h1 ^= k1;
489
490        h1 = h1.rotate_left(27);
491        h1 = h1.wrapping_add(h2);
492        h1 = h1.wrapping_mul(5);
493        h1 = h1.wrapping_add(0x52dce729);
494
495        k2 = k2.wrapping_mul(c2);
496        k2 = k2.rotate_left(33);
497        k2 = k2.wrapping_mul(c1);
498        h2 ^= k2;
499
500        h2 = h2.rotate_left(31);
501        h2 = h2.wrapping_add(h1);
502        h2 = h2.wrapping_mul(5);
503        h2 = h2.wrapping_add(0x38495ab5);
504    }
505    let (mut k1, mut k2) = (0u64, 0u64);
506
507    if len & 15 == 15 {
508        k2 ^= (bytes[(block_count * read_size) as usize + 14] as u64) << 48;
509    }
510    if len & 15 >= 14 {
511        k2 ^= (bytes[(block_count * read_size) as usize + 13] as u64) << 40;
512    }
513    if len & 15 >= 13 {
514        k2 ^= (bytes[(block_count * read_size) as usize + 12] as u64) << 32;
515    }
516    if len & 15 >= 12 {
517        k2 ^= (bytes[(block_count * read_size) as usize + 11] as u64) << 24;
518    }
519    if len & 15 >= 11 {
520        k2 ^= (bytes[(block_count * read_size) as usize + 10] as u64) << 16;
521    }
522    if len & 15 >= 10 {
523        k2 ^= (bytes[(block_count * read_size) as usize + 9] as u64) << 8;
524    }
525    if len & 15 >= 9 {
526        k2 ^= bytes[(block_count * read_size) as usize + 8] as u64;
527        k2 = k2.wrapping_mul(c2);
528        k2 = k2.rotate_left(33);
529        k2 = k2.wrapping_mul(c1);
530        h2 ^= k2;
531    }
532
533    if len & 15 >= 8 {
534        k1 ^= (bytes[(block_count * read_size) as usize + 7] as u64) << 56;
535    }
536    if len & 15 >= 7 {
537        k1 ^= (bytes[(block_count * read_size) as usize + 6] as u64) << 48;
538    }
539    if len & 15 >= 6 {
540        k1 ^= (bytes[(block_count * read_size) as usize + 5] as u64) << 40;
541    }
542    if len & 15 >= 5 {
543        k1 ^= (bytes[(block_count * read_size) as usize + 4] as u64) << 32;
544    }
545    if len & 15 >= 4 {
546        k1 ^= (bytes[(block_count * read_size) as usize + 3] as u64) << 24;
547    }
548    if len & 15 >= 3 {
549        k1 ^= (bytes[(block_count * read_size) as usize + 2] as u64) << 16;
550    }
551    if len & 15 >= 2 {
552        k1 ^= (bytes[(block_count * read_size) as usize + 1] as u64) << 8;
553    }
554    if len & 15 >= 1 {
555        k1 ^= bytes[(block_count * read_size) as usize] as u64;
556        k1 = k1.wrapping_mul(c1);
557        k1 = k1.rotate_left(31);
558        k1 = k1.wrapping_mul(c2);
559        h1 ^= k1;
560    }
561
562    h1 ^= bytes.len() as u64;
563    h2 ^= bytes.len() as u64;
564
565    h1 = h1.wrapping_add(h2);
566    h2 = h2.wrapping_add(h1);
567
568    h1 = fmix64(h1);
569    h2 = fmix64(h2);
570
571    h1 = h1.wrapping_add(h2);
572    h2 = h2.wrapping_add(h1);
573
574    (h1, h2)
575}
576
577#[cfg(test)]
578mod test_hash {
579    use super::murmurhash3_x64_128;
580
581    #[test]
582    fn test_empty_string() {
583        assert!(murmurhash3_x64_128("".as_bytes(), 0) == (0, 0));
584    }
585
586    #[test]
587    fn test_tail_lengths() {
588        assert!(
589            murmurhash3_x64_128("1".as_bytes(), 0) == (8213365047359667313, 10676604921780958775)
590        );
591        assert!(
592            murmurhash3_x64_128("12".as_bytes(), 0) == (5355690773644049813, 9855895140584599837)
593        );
594        assert!(
595            murmurhash3_x64_128("123".as_bytes(), 0) == (10978418110857903978, 4791445053355511657)
596        );
597        assert!(
598            murmurhash3_x64_128("1234".as_bytes(), 0) == (619023178690193332, 3755592904005385637)
599        );
600        assert!(
601            murmurhash3_x64_128("12345".as_bytes(), 0)
602                == (2375712675693977547, 17382870096830835188)
603        );
604        assert!(
605            murmurhash3_x64_128("123456".as_bytes(), 0)
606                == (16435832985690558678, 5882968373513761278)
607        );
608        assert!(
609            murmurhash3_x64_128("1234567".as_bytes(), 0)
610                == (3232113351312417698, 4025181827808483669)
611        );
612        assert!(
613            murmurhash3_x64_128("12345678".as_bytes(), 0)
614                == (4272337174398058908, 10464973996478965079)
615        );
616        assert!(
617            murmurhash3_x64_128("123456789".as_bytes(), 0)
618                == (4360720697772133540, 11094893415607738629)
619        );
620        assert!(
621            murmurhash3_x64_128("123456789a".as_bytes(), 0)
622                == (12594836289594257748, 2662019112679848245)
623        );
624        assert!(
625            murmurhash3_x64_128("123456789ab".as_bytes(), 0)
626                == (6978636991469537545, 12243090730442643750)
627        );
628        assert!(
629            murmurhash3_x64_128("123456789abc".as_bytes(), 0)
630                == (211890993682310078, 16480638721813329343)
631        );
632        assert!(
633            murmurhash3_x64_128("123456789abcd".as_bytes(), 0)
634                == (12459781455342427559, 3193214493011213179)
635        );
636        assert!(
637            murmurhash3_x64_128("123456789abcde".as_bytes(), 0)
638                == (12538342858731408721, 9820739847336455216)
639        );
640        assert!(
641            murmurhash3_x64_128("123456789abcdef".as_bytes(), 0)
642                == (9165946068217512774, 2451472574052603025)
643        );
644        assert!(
645            murmurhash3_x64_128("123456789abcdef1".as_bytes(), 0)
646                == (9259082041050667785, 12459473952842597282)
647        );
648    }
649
650    #[test]
651    fn test_large_data() {
652        assert!(murmurhash3_x64_128("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam at consequat massa. Cras eleifend pellentesque ex, at dignissim libero maximus ut. Sed eget nulla felis".as_bytes(), 0)
653            == (9455322759164802692, 17863277201603478371));
654    }
655
656    #[test]
657    fn test_unaligned_full_block() {
658        let data = b"x123456789abcdef1";
659        assert!(murmurhash3_x64_128(&data[1..], 0) == (9259082041050667785, 12459473952842597282));
660    }
661}
662
663#[cfg(test)]
664mod case_tests {
665    use super::*;
666
667    #[test]
668    fn test_to_snake_case() {
669        assert_eq!(to_snake_case("camelCase"), "camel_case");
670        assert_eq!(to_snake_case("PascalCase"), "pascal_case");
671        assert_eq!(to_snake_case("HTTPRequest"), "http_request");
672        assert_eq!(to_snake_case("simpleTest"), "simple_test");
673        assert_eq!(to_snake_case("already_snake"), "already_snake");
674        assert_eq!(to_snake_case("ABC"), "abc");
675    }
676
677    #[test]
678    fn test_to_camel_case() {
679        assert_eq!(to_camel_case("snake_case"), "snakeCase");
680        assert_eq!(to_camel_case("simple_test"), "simpleTest");
681        assert_eq!(to_camel_case("already"), "already");
682        assert_eq!(to_camel_case("a_b_c"), "aBC");
683    }
684}
685
686pub mod buffer_rw_string {
687    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
688    use std::arch::aarch64::*;
689    #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
690    use std::arch::x86_64::*;
691    #[cfg(all(
692        any(target_arch = "x86", target_arch = "x86_64"),
693        target_feature = "sse2",
694        not(target_feature = "avx2")
695    ))]
696    use std::arch::x86_64::*;
697
698    use crate::buffer::{Reader, Writer};
699    use crate::error::Error;
700
701    #[inline]
702    pub fn write_latin1_standard(writer: &mut Writer, s: &str) {
703        for c in s.chars() {
704            let b = c as u32;
705            assert!(b <= 0xFF, "Non-Latin1 character found");
706            writer.write_u8(b as u8);
707        }
708    }
709
710    #[inline(always)]
711    pub fn write_latin1_string(writer: &mut Writer, s: &str) {
712        if s.len() < 128 {
713            // Fast path for small buffers
714            let bytes = s.as_bytes();
715            // CRITICAL: Only safe if ASCII (UTF-8 == Latin1 for ASCII)
716            let is_ascii = bytes.iter().all(|&b| b < 0x80);
717            if is_ascii {
718                writer.bf.reserve(s.len());
719                writer.bf.extend_from_slice(bytes);
720            } else {
721                // Non-ASCII: must iterate chars to extract Latin1 byte values
722                writer.bf.reserve(s.len());
723                for c in s.chars() {
724                    let v = c as u32;
725                    assert!(v <= 0xFF, "Non-Latin1 character found");
726                    writer.bf.push(v as u8);
727                }
728            }
729            return;
730        }
731        write_latin1_simd(writer, s);
732    }
733
734    #[inline]
735    pub fn write_utf8_standard(writer: &mut Writer, s: &str) {
736        let bytes = s.as_bytes();
737        writer.bf.extend_from_slice(bytes);
738    }
739
740    #[inline]
741    pub fn write_utf16_standard(writer: &mut Writer, utf16: &[u16]) {
742        #[cfg(target_endian = "little")]
743        {
744            let total_bytes = utf16.len() * 2;
745            let old_len = writer.bf.len();
746            writer.bf.reserve(total_bytes);
747            unsafe {
748                let dest = writer.bf.as_mut_ptr().add(old_len);
749                let src = utf16.as_ptr() as *const u8;
750                std::ptr::copy_nonoverlapping(src, dest, total_bytes);
751                writer.bf.set_len(old_len + total_bytes);
752            }
753        }
754        #[cfg(target_endian = "big")]
755        {
756            let total_bytes = utf16.len() * 2;
757            let old_len = writer.bf.len();
758            writer.bf.reserve(total_bytes);
759            unsafe {
760                let dest = writer.bf.as_mut_ptr().add(old_len);
761                // Need to swap bytes for each u16 to little-endian
762                for (i, &unit) in utf16.iter().enumerate() {
763                    let swapped = unit.swap_bytes();
764                    let ptr = dest.add(i * 2) as *mut u16;
765                    std::ptr::write_unaligned(ptr, swapped);
766                }
767                writer.bf.set_len(old_len + total_bytes);
768            }
769        }
770    }
771
772    #[inline]
773    pub fn read_latin1_standard(reader: &mut Reader, len: usize) -> Result<String, Error> {
774        let slice = reader.sub_slice(reader.get_cursor(), reader.get_cursor() + len)?;
775        let result: String = slice.iter().map(|&b| b as char).collect();
776        reader.move_next(len);
777        Ok(result)
778    }
779
780    #[inline]
781    pub fn read_utf8_standard(reader: &mut Reader, len: usize) -> Result<String, Error> {
782        let slice = reader.sub_slice(reader.get_cursor(), reader.get_cursor() + len)?;
783        // Rust is the only runtime that checks UTF-8 string payloads by default; borrow first so
784        // the check adds no temporary Vec before constructing the final String.
785        let value = std::str::from_utf8(slice)
786            .map_err(|_| Error::encoding_error("invalid UTF-8 string"))?
787            .to_owned();
788        reader.move_next(len);
789        Ok(value)
790    }
791
792    #[inline]
793    pub fn read_utf16_standard(reader: &mut Reader, len: usize) -> Result<String, Error> {
794        if len % 2 != 0 {
795            return Err(Error::encoding_error("UTF-16 length must be even"));
796        }
797        unsafe {
798            let slice = std::slice::from_raw_parts(reader.bf.as_ptr().add(reader.cursor), len);
799            let units: Vec<u16> = slice
800                .chunks_exact(2)
801                .map(|c| u16::from_le_bytes([c[0], c[1]]))
802                .collect();
803            reader.move_next(len);
804            Ok(String::from_utf16_lossy(&units))
805        }
806    }
807
808    #[inline]
809    fn is_ascii_bytes(bytes: &[u8]) -> bool {
810        let len = bytes.len();
811        let mut i = 0;
812
813        #[cfg(target_arch = "x86_64")]
814        unsafe {
815            if is_x86_feature_detected!("avx2") && len >= 32 {
816                while i + 32 <= len {
817                    let chunk = _mm256_loadu_si256(bytes.as_ptr().add(i) as *const __m256i);
818                    let mask = _mm256_movemask_epi8(chunk);
819                    if mask != 0 {
820                        return false;
821                    }
822                    i += 32;
823                }
824            }
825        }
826
827        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
828        unsafe {
829            if is_x86_feature_detected!("sse2") && len >= 16 {
830                while i + 16 <= len {
831                    let chunk = _mm_loadu_si128(bytes.as_ptr().add(i) as *const __m128i);
832                    let mask = _mm_movemask_epi8(chunk);
833                    if mask != 0 {
834                        return false;
835                    }
836                    i += 16;
837                }
838            }
839        }
840
841        #[cfg(target_arch = "aarch64")]
842        unsafe {
843            if std::arch::is_aarch64_feature_detected!("neon") && len >= 16 {
844                while i + 16 <= len {
845                    let chunk = vld1q_u8(bytes.as_ptr().add(i));
846                    if vmaxvq_u8(chunk) >= 0x80 {
847                        return false;
848                    }
849                    i += 16;
850                }
851            }
852        }
853
854        // Scalar fallback
855        bytes[i..].iter().all(|&b| b < 0x80)
856    }
857
858    #[inline]
859    pub fn write_latin1_simd(writer: &mut Writer, s: &str) {
860        if s.is_empty() {
861            return;
862        }
863
864        let bytes = s.as_bytes();
865
866        // CRITICAL OPTIMIZATION: For ASCII strings, UTF-8 bytes == Latin1 bytes
867        // Check if all ASCII using SIMD
868        if is_ascii_bytes(bytes) {
869            // Zero-copy fast path: direct write
870            let len = bytes.len();
871            writer.bf.reserve(len);
872            writer.bf.extend_from_slice(bytes);
873        } else {
874            // Non-ASCII: Must iterate chars to extract Latin1 byte values
875            // Example: 'À' in Rust String is UTF-8 [0xC3, 0x80] but Latin1 is [0xC0]
876            let mut buf: Vec<u8> = Vec::with_capacity(s.len());
877            for c in s.chars() {
878                let v = c as u32;
879                assert!(v <= 0xFF, "Non-Latin1 character found");
880                buf.push(v as u8);
881            }
882            let len = buf.len();
883            writer.bf.reserve(len);
884            writer.bf.extend_from_slice(&buf);
885        }
886    }
887
888    #[inline]
889    pub fn read_latin1_simd(reader: &mut Reader, len: usize) -> Result<String, Error> {
890        if len == 0 {
891            return Ok(String::new());
892        }
893        let src = reader.sub_slice(reader.get_cursor(), reader.get_cursor() + len)?;
894
895        // Pessimistic allocation: Latin1 0x80-0xFF expands to 2 bytes in UTF-8
896        let mut out: Vec<u8> = Vec::with_capacity(len * 2);
897
898        unsafe {
899            let out_ptr = out.as_mut_ptr();
900            let mut out_len = 0usize;
901            let mut i = 0usize;
902
903            // ---- AVX2 fast-path: process 32 ASCII bytes at once ----
904            #[cfg(target_arch = "x86_64")]
905            {
906                if std::arch::is_x86_feature_detected!("avx2") {
907                    use std::arch::x86_64::*;
908                    while i + 32 <= len {
909                        let ptr = src.as_ptr().add(i) as *const __m256i;
910                        let chunk = _mm256_loadu_si256(ptr);
911                        let mask = _mm256_movemask_epi8(chunk);
912                        if mask == 0 {
913                            // All ASCII: direct copy (no conversion needed)
914                            _mm256_storeu_si256(out_ptr.add(out_len) as *mut __m256i, chunk);
915                            out_len += 32;
916                            i += 32;
917                            continue;
918                        } else {
919                            // Contains Latin1 bytes, break to scalar
920                            break;
921                        }
922                    }
923                }
924            }
925
926            // ---- SSE2 fast-path: process 16 ASCII bytes at once ----
927            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
928            {
929                if std::arch::is_x86_feature_detected!("sse2") {
930                    use std::arch::x86_64::*;
931                    while i + 16 <= len {
932                        let ptr = src.as_ptr().add(i) as *const __m128i;
933                        let chunk = _mm_loadu_si128(ptr);
934                        let mask = _mm_movemask_epi8(chunk);
935                        if mask == 0 {
936                            // All ASCII: direct copy
937                            _mm_storeu_si128(out_ptr.add(out_len) as *mut __m128i, chunk);
938                            out_len += 16;
939                            i += 16;
940                            continue;
941                        } else {
942                            break;
943                        }
944                    }
945                }
946            }
947
948            // ---- NEON fast-path: process 16 ASCII bytes at once ----
949            #[cfg(target_arch = "aarch64")]
950            {
951                if std::arch::is_aarch64_feature_detected!("neon") {
952                    use std::arch::aarch64::*;
953                    while i + 16 <= len {
954                        let ptr = src.as_ptr().add(i);
955                        let v = vld1q_u8(ptr);
956                        // Check if any byte >= 0x80
957                        if vmaxvq_u8(v) < 0x80 {
958                            // All ASCII: direct copy
959                            vst1q_u8(out_ptr.add(out_len), v);
960                            out_len += 16;
961                            i += 16;
962                            continue;
963                        } else {
964                            break;
965                        }
966                    }
967                }
968            }
969
970            // ---- Scalar fallback: convert Latin1 -> UTF-8 ----
971            // ASCII (0x00-0x7F): copy as-is
972            // Latin1 (0x80-0xFF): encode as 2-byte UTF-8
973            while i < len {
974                let b = *src.get_unchecked(i);
975                if b < 0x80 {
976                    *out_ptr.add(out_len) = b;
977                    out_len += 1;
978                } else {
979                    // Latin1 byte 0x80-0xFF -> UTF-8 encoding
980                    // Example: 0xC0 (À) -> [0xC3, 0x80]
981                    *out_ptr.add(out_len) = 0xC0 | (b >> 6);
982                    *out_ptr.add(out_len + 1) = 0x80 | (b & 0x3F);
983                    out_len += 2;
984                }
985                i += 1;
986            }
987
988            out.set_len(out_len);
989        }
990        reader.move_next(len);
991        Ok(unsafe { String::from_utf8_unchecked(out) })
992    }
993
994    #[cfg(test)]
995    mod tests {
996        use super::*;
997        use crate::buffer::{Reader, Writer};
998
999        #[test]
1000        fn test_latin1() {
1001            let samples = [
1002                "Hello World!",
1003                "Rusty Café",
1004                "1234567890",
1005                "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝ",
1006            ];
1007
1008            for s in samples {
1009                let mut buffer = vec![];
1010                let mut writer = Writer::from_buffer(&mut buffer);
1011                write_latin1_simd(&mut writer, s);
1012                write_latin1_simd(&mut writer, s);
1013                let bytes = &*writer.dump();
1014                let bytes_len = bytes.len() / 2;
1015                let mut reader = Reader::new(bytes);
1016                assert_eq!(read_latin1_standard(&mut reader, bytes_len).unwrap(), s);
1017                assert_eq!(read_latin1_standard(&mut reader, bytes_len).unwrap(), s);
1018
1019                let mut buffer = vec![];
1020                let mut writer = Writer::from_buffer(&mut buffer);
1021                write_latin1_standard(&mut writer, s);
1022                write_latin1_standard(&mut writer, s);
1023                let bytes = &*writer.dump();
1024                let bytes_len = bytes.len() / 2;
1025                let mut reader = Reader::new(bytes);
1026                assert_eq!(read_latin1_simd(&mut reader, bytes_len).unwrap(), s);
1027                assert_eq!(read_latin1_simd(&mut reader, bytes_len).unwrap(), s);
1028            }
1029        }
1030
1031        #[test]
1032        fn test_utf8() {
1033            let samples = [
1034                "hello",
1035                "rust语言",
1036                "你好，世界",
1037                "emoji 😀😃😄😁",
1038                "mixed ASCII + 中文 + emoji 😁",
1039            ];
1040
1041            for s in samples {
1042                let bytes_len = s.len();
1043
1044                let mut buffer = vec![];
1045                let mut writer = Writer::from_buffer(&mut buffer);
1046                write_utf8_standard(&mut writer, s);
1047                write_utf8_standard(&mut writer, s);
1048                let bytes = &*writer.dump();
1049                let mut reader = Reader::new(bytes);
1050                assert_eq!(read_utf8_standard(&mut reader, bytes_len).unwrap(), s);
1051                assert_eq!(read_utf8_standard(&mut reader, bytes_len).unwrap(), s);
1052            }
1053        }
1054
1055        #[test]
1056        fn test_utf16() {
1057            let samples = [
1058                "hello",
1059                "rust语言",
1060                "你好，世界",
1061                "emoji 😀😃😄😁",
1062                "混合文字 + emoji 🐍💻🦀",
1063            ];
1064            for s in samples {
1065                let utf16: Vec<u16> = s.encode_utf16().collect();
1066                let bytes_len = utf16.len() * 2;
1067
1068                let mut buffer = vec![];
1069                let mut writer = Writer::from_buffer(&mut buffer);
1070                write_utf16_standard(&mut writer, &utf16);
1071                write_utf16_standard(&mut writer, &utf16);
1072
1073                let mut buffer = vec![];
1074                let mut writer = Writer::from_buffer(&mut buffer);
1075                write_utf16_standard(&mut writer, &utf16);
1076                write_utf16_standard(&mut writer, &utf16);
1077                let bytes = &*writer.dump();
1078                let mut reader = Reader::new(bytes);
1079                assert_eq!(read_utf16_standard(&mut reader, bytes_len).unwrap(), s);
1080                assert_eq!(read_utf16_standard(&mut reader, bytes_len).unwrap(), s);
1081            }
1082        }
1083    }
1084}
fory_core/util/string_util.rs

fory_core/util/
string_util.rs
