fory_core/util/
string_util.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::ptr;
19
20const MAX_HASH32: u64 = (1 << 31) - 1;
21
22#[allow(dead_code)]
23pub enum StringFlag {
24    LATIN1 = 0,
25    UTF8 = 1,
26}
27
28/// Swaps the high 8 bits and the low 8 bits of a 16-bit value.
29fn swap_endian(value: u16) -> u16 {
30    value.rotate_right(8)
31}
32
33/// Converts UTF-16 encoded data to UTF-8.
34pub fn to_utf8(utf16: &[u16], is_little_endian: bool) -> Result<Vec<u8>, String> {
35    // Pre-allocating capacity to avoid dynamic resizing.
36    // Longest case: 1 u16 to 3 u8.
37    let mut utf8_bytes: Vec<u8> = Vec::with_capacity(utf16.len() * 3);
38    let ptr = utf8_bytes.as_mut_ptr();
39    let mut offset = 0;
40    let mut iter = utf16.iter();
41    while let Some(&wc) = iter.next() {
42        let wc = if is_little_endian {
43            swap_endian(wc)
44        } else {
45            wc
46        };
47        match wc {
48            code_point if code_point < 0x80 => {
49                unsafe {
50                    ptr.add(offset).write(code_point as u8);
51                }
52                offset += 1;
53            }
54            code_point if code_point < 0x800 => {
55                let bytes = [
56                    ((code_point >> 6) & 0b1_1111) as u8 | 0b1100_0000,
57                    (code_point & 0b11_1111) as u8 | 0b1000_0000,
58                ];
59                unsafe {
60                    ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), 2);
61                }
62                offset += 2;
63            }
64            wc1 if (0xd800..=0xdbff).contains(&wc1) => {
65                if let Some(&wc2) = iter.next() {
66                    let wc2 = if is_little_endian {
67                        swap_endian(wc2)
68                    } else {
69                        wc2
70                    };
71                    if !(0xdc00..=0xdfff).contains(&wc2) {
72                        return Err("Invalid UTF-16 string: wrong surrogate pair".to_string());
73                    }
74                    let code_point =
75                        ((((wc1 as u32) - 0xd800) << 10) | ((wc2 as u32) - 0xdc00)) + 0x10000;
76                    let bytes = [
77                        ((code_point >> 18) & 0b111) as u8 | 0b1111_0000,
78                        ((code_point >> 12) & 0b11_1111) as u8 | 0b1000_0000,
79                        ((code_point >> 6) & 0b11_1111) as u8 | 0b1000_0000,
80                        (code_point & 0b11_1111) as u8 | 0b1000_0000,
81                    ];
82                    unsafe {
83                        ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), 4);
84                    }
85                    offset += 4;
86                } else {
87                    return Err("Invalid UTF-16 string: missing surrogate pair".to_string());
88                }
89            }
90            _ => {
91                let bytes = [
92                    ((wc >> 12) | 0b1110_0000) as u8,
93                    ((wc >> 6) & 0b11_1111) as u8 | 0b1000_0000,
94                    (wc & 0b11_1111) as u8 | 0b1000_0000,
95                ];
96                unsafe {
97                    ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), 3);
98                }
99                offset += 3;
100            }
101        }
102    }
103    unsafe {
104        utf8_bytes.set_len(offset);
105    }
106    Ok(utf8_bytes)
107}
108
109/// Converts a camelCase or PascalCase string to snake_case.
110pub fn to_snake_case(name: &str) -> String {
111    let mut result = String::with_capacity(name.len() + 4);
112    let chars: Vec<char> = name.chars().collect();
113
114    for (i, &c) in chars.iter().enumerate() {
115        if c.is_ascii_uppercase() {
116            if i > 0 {
117                let prev_upper = chars.get(i - 1).is_some_and(|c| c.is_ascii_uppercase());
118                let next_upper_or_end = chars.get(i + 1).map_or(true, |c| c.is_ascii_uppercase());
119                if !prev_upper || !next_upper_or_end {
120                    result.push('_');
121                }
122            }
123            result.push(c.to_ascii_lowercase());
124        } else {
125            result.push(c);
126        }
127    }
128    result
129}
130
131/// Converts a snake_case string to lowerCamelCase.
132pub fn to_camel_case(name: &str) -> String {
133    let mut result = String::with_capacity(name.len());
134    let mut capitalize_next = false;
135
136    for c in name.chars() {
137        if c == '_' {
138            capitalize_next = true;
139        } else if capitalize_next {
140            result.push(c.to_ascii_uppercase());
141            capitalize_next = false;
142        } else {
143            result.push(c);
144        }
145    }
146    result
147}
148
149#[allow(dead_code)]
150pub fn compute_string_hash(s: &str) -> u32 {
151    let mut hash: u64 = 17;
152    s.as_bytes().iter().for_each(|b| {
153        hash = (hash * 31) + (*b as u64);
154        while hash >= MAX_HASH32 {
155            hash /= 7;
156        }
157    });
158    hash as u32
159}
160
161#[cfg(target_feature = "neon")]
162use std::arch::aarch64::*;
163
164#[cfg(target_feature = "avx2")]
165use std::arch::x86_64::*;
166
167#[cfg(target_feature = "sse2")]
168use std::arch::x86_64::*;
169
170#[cfg(target_arch = "x86_64")]
171pub const MIN_DIM_SIZE_AVX: usize = 32;
172
173#[cfg(any(
174    target_arch = "x86",
175    target_arch = "x86_64",
176    all(target_arch = "aarch64", target_feature = "neon")
177))]
178pub const MIN_DIM_SIZE_SIMD: usize = 16;
179
180#[cfg(target_arch = "x86_64")]
181unsafe fn is_latin_avx(s: &str) -> bool {
182    let bytes = s.as_bytes();
183    let len = bytes.len();
184    let mut i = 0;
185    // SIMD skip ASCII
186    while i + MIN_DIM_SIZE_AVX <= len {
187        let chunk = _mm256_loadu_si256(bytes.as_ptr().add(i) as *const __m256i);
188        let hi_mask = _mm256_set1_epi8(0x80u8 as i8);
189        let masked = _mm256_and_si256(chunk, hi_mask);
190        let cmp = _mm256_cmpeq_epi8(masked, _mm256_setzero_si256());
191        if _mm256_movemask_epi8(cmp) != -1 {
192            break;
193        }
194        i += MIN_DIM_SIZE_AVX;
195    }
196    // check latin in remaining chars
197    let s_tail = &s[i..];
198    for c in s_tail.chars() {
199        if c as u32 > 0xFF {
200            return false;
201        }
202    }
203    true
204}
205
206#[cfg(target_feature = "sse2")]
207unsafe fn is_latin_sse(s: &str) -> bool {
208    let bytes = s.as_bytes();
209    let len = bytes.len();
210    let mut i = 0;
211    // SIMD skip ASCII
212    while i + MIN_DIM_SIZE_SIMD <= len {
213        let chunk = _mm_loadu_si128(bytes.as_ptr().add(i) as *const __m128i);
214        let hi_mask = _mm_set1_epi8(0x80u8 as i8);
215        let masked = _mm_and_si128(chunk, hi_mask);
216        let cmp = _mm_cmpeq_epi8(masked, _mm_setzero_si128());
217        if _mm_movemask_epi8(cmp) != 0xFFFF {
218            break;
219        }
220        i += MIN_DIM_SIZE_SIMD;
221    }
222    // check latin in remaining chars
223    let s_tail = &s[i..];
224    for c in s_tail.chars() {
225        if c as u32 > 0xFF {
226            return false;
227        }
228    }
229    true
230}
231
232#[cfg(target_feature = "neon")]
233unsafe fn is_latin_neon(s: &str) -> bool {
234    let bytes = s.as_bytes();
235    let len = bytes.len();
236    let mut i = 0;
237    // SIMD skip ASCII
238    while i + MIN_DIM_SIZE_SIMD <= len {
239        let chunk = vld1q_u8(bytes.as_ptr().add(i));
240        let hi_mask = vdupq_n_u8(0x80);
241        let masked = vandq_u8(chunk, hi_mask);
242        if vmaxvq_u8(masked) != 0 {
243            break;
244        }
245        i += MIN_DIM_SIZE_SIMD;
246    }
247    // check latin in remaining chars
248    let s_tail = &s[i..];
249    for c in s_tail.chars() {
250        if c as u32 > 0xFF {
251            return false;
252        }
253    }
254    true
255}
256
257fn is_latin_standard(s: &str) -> bool {
258    s.chars().all(|c| c as u32 <= 0xFF)
259}
260
261pub fn is_latin(s: &str) -> bool {
262    #[cfg(target_arch = "x86_64")]
263    {
264        if is_x86_feature_detected!("avx")
265            && is_x86_feature_detected!("fma")
266            && s.len() >= MIN_DIM_SIZE_AVX
267        {
268            return unsafe { is_latin_avx(s) };
269        }
270    }
271
272    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
273    {
274        if is_x86_feature_detected!("sse") && s.len() >= MIN_DIM_SIZE_SIMD {
275            return unsafe { is_latin_sse(s) };
276        }
277    }
278
279    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
280    {
281        if std::arch::is_aarch64_feature_detected!("neon") && s.len() >= MIN_DIM_SIZE_SIMD {
282            return unsafe { is_latin_neon(s) };
283        }
284    }
285    is_latin_standard(s)
286}
287
288#[cfg(target_arch = "x86_64")]
289unsafe fn get_latin1_length_avx(s: &str) -> i32 {
290    let bytes = s.as_bytes();
291    let len = bytes.len();
292    let mut count = 0;
293    // SIMD skip ASCII
294    while count + MIN_DIM_SIZE_AVX <= len {
295        let chunk = _mm256_loadu_si256(bytes.as_ptr().add(count) as *const __m256i);
296        let hi_mask = _mm256_set1_epi8(0x80u8 as i8);
297        let masked = _mm256_and_si256(chunk, hi_mask);
298        let cmp = _mm256_cmpeq_epi8(masked, _mm256_setzero_si256());
299        if _mm256_movemask_epi8(cmp) != -1 {
300            break;
301        }
302        count += MIN_DIM_SIZE_AVX;
303    }
304    // check latin in remaining chars
305    let s_tail = &s[count..];
306    for c in s_tail.chars() {
307        if c as u32 > 0xFF {
308            return -1;
309        }
310        count += 1;
311    }
312    count as i32
313}
314
315#[cfg(target_feature = "sse2")]
316unsafe fn get_latin1_length_sse(s: &str) -> i32 {
317    let bytes = s.as_bytes();
318    let len = bytes.len();
319    let mut count = 0;
320    // SIMD skip ASCII
321    while count + MIN_DIM_SIZE_SIMD <= len {
322        let chunk = _mm_loadu_si128(bytes.as_ptr().add(count) as *const __m128i);
323        let hi_mask = _mm_set1_epi8(0x80u8 as i8);
324        let masked = _mm_and_si128(chunk, hi_mask);
325        let cmp = _mm_cmpeq_epi8(masked, _mm_setzero_si128());
326        if _mm_movemask_epi8(cmp) != 0xFFFF {
327            break;
328        }
329        count += MIN_DIM_SIZE_SIMD;
330    }
331    // check latin in remaining chars
332    let s_tail = &s[count..];
333    for c in s_tail.chars() {
334        if c as u32 > 0xFF {
335            return -1;
336        }
337        count += 1;
338    }
339    count as i32
340}
341
342#[cfg(target_feature = "neon")]
343unsafe fn get_latin1_length_neon(s: &str) -> i32 {
344    let bytes = s.as_bytes();
345    let len = bytes.len();
346    let mut count = 0;
347    // SIMD skip ASCII
348    while count + MIN_DIM_SIZE_SIMD <= len {
349        let chunk = vld1q_u8(bytes.as_ptr().add(count));
350        let hi_mask = vdupq_n_u8(0x80);
351        let masked = vandq_u8(chunk, hi_mask);
352        if vmaxvq_u8(masked) != 0 {
353            break;
354        }
355        count += MIN_DIM_SIZE_SIMD;
356    }
357    // check latin in remaining chars
358    let s_tail = &s[count..];
359    for c in s_tail.chars() {
360        if c as u32 > 0xFF {
361            return -1;
362        }
363        count += 1;
364    }
365    count as i32
366}
367
368fn get_latin1_length_standard(s: &str) -> i32 {
369    let mut count = 0;
370    for c in s.chars() {
371        if c as u32 > 0xFF {
372            return -1;
373        }
374        count += 1;
375    }
376    count
377}
378
379pub fn get_latin1_length(s: &str) -> i32 {
380    #[cfg(target_arch = "x86_64")]
381    {
382        if is_x86_feature_detected!("avx")
383            && is_x86_feature_detected!("fma")
384            && s.len() >= MIN_DIM_SIZE_AVX
385        {
386            return unsafe { get_latin1_length_avx(s) };
387        }
388    }
389
390    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
391    {
392        if is_x86_feature_detected!("sse") && s.len() >= MIN_DIM_SIZE_SIMD {
393            return unsafe { get_latin1_length_sse(s) };
394        }
395    }
396
397    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
398    {
399        if std::arch::is_aarch64_feature_detected!("neon") && s.len() >= MIN_DIM_SIZE_SIMD {
400            return unsafe { get_latin1_length_neon(s) };
401        }
402    }
403    get_latin1_length_standard(s)
404}
405
406#[cfg(test)]
407mod latin_tests {
408    // Import content from external modules
409    use super::*;
410    use rand::Rng;
411
412    fn generate_random_string(length: usize) -> String {
413        const CHARSET: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
414        let mut rng = rand::thread_rng();
415
416        let result: String = (0..length)
417            .map(|_| {
418                let idx = rng.gen_range(0..CHARSET.len());
419                CHARSET[idx] as char
420            })
421            .collect();
422
423        result
424    }
425
426    #[test]
427    fn test_is_latin() {
428        let s = generate_random_string(1000);
429        let not_latin_str = generate_random_string(1000) + "abc\u{1234}";
430
431        #[cfg(target_arch = "x86_64")]
432        {
433            if is_x86_feature_detected!("avx") && is_x86_feature_detected!("fma") {
434                assert!(unsafe { is_latin_avx(&s) });
435                assert!(!unsafe { is_latin_avx(&not_latin_str) });
436            }
437        }
438
439        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
440        {
441            if is_x86_feature_detected!("sse") && s.len() >= MIN_DIM_SIZE_SIMD {
442                assert!(unsafe { is_latin_sse(&s) });
443                assert!(!unsafe { is_latin_sse(&not_latin_str) });
444            }
445        }
446
447        #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
448        {
449            if std::arch::is_aarch64_feature_detected!("neon") && s.len() >= MIN_DIM_SIZE_SIMD {
450                assert!(unsafe { is_latin_neon(&s) });
451                assert!(!unsafe { is_latin_neon(&not_latin_str) });
452            }
453        }
454        assert!(is_latin_standard(&s));
455        assert!(!is_latin_standard(&not_latin_str));
456    }
457}
458
459fn fmix64(mut k: u64) -> u64 {
460    k ^= k >> 33;
461    k = k.wrapping_mul(0xff51afd7ed558ccdu64);
462    k ^= k >> 33;
463    k = k.wrapping_mul(0xc4ceb9fe1a85ec53u64);
464    k ^= k >> 33;
465
466    k
467}
468
469pub fn murmurhash3_x64_128(bytes: &[u8], seed: u64) -> (u64, u64) {
470    let c1 = 0x87c37b91114253d5u64;
471    let c2 = 0x4cf5ad432745937fu64;
472    let read_size = 16;
473    let len = bytes.len() as u64;
474    let block_count = len / read_size;
475
476    let (mut h1, mut h2) = (seed, seed);
477
478    for i in 0..block_count as usize {
479        let offset = i * read_size as usize;
480        // Input byte slices can be arbitrarily aligned. Read MurmurHash3 blocks
481        // as explicit little-endian bytes instead of forming an aligned u64 slice.
482        let mut k1 = u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap());
483        let mut k2 = u64::from_le_bytes(bytes[offset + 8..offset + 16].try_into().unwrap());
484
485        k1 = k1.wrapping_mul(c1);
486        k1 = k1.rotate_left(31);
487        k1 = k1.wrapping_mul(c2);
488        h1 ^= k1;
489
490        h1 = h1.rotate_left(27);
491        h1 = h1.wrapping_add(h2);
492        h1 = h1.wrapping_mul(5);
493        h1 = h1.wrapping_add(0x52dce729);
494
495        k2 = k2.wrapping_mul(c2);
496        k2 = k2.rotate_left(33);
497        k2 = k2.wrapping_mul(c1);
498        h2 ^= k2;
499
500        h2 = h2.rotate_left(31);
501        h2 = h2.wrapping_add(h1);
502        h2 = h2.wrapping_mul(5);
503        h2 = h2.wrapping_add(0x38495ab5);
504    }
505    let (mut k1, mut k2) = (0u64, 0u64);
506
507    if len & 15 == 15 {
508        k2 ^= (bytes[(block_count * read_size) as usize + 14] as u64) << 48;
509    }
510    if len & 15 >= 14 {
511        k2 ^= (bytes[(block_count * read_size) as usize + 13] as u64) << 40;
512    }
513    if len & 15 >= 13 {
514        k2 ^= (bytes[(block_count * read_size) as usize + 12] as u64) << 32;
515    }
516    if len & 15 >= 12 {
517        k2 ^= (bytes[(block_count * read_size) as usize + 11] as u64) << 24;
518    }
519    if len & 15 >= 11 {
520        k2 ^= (bytes[(block_count * read_size) as usize + 10] as u64) << 16;
521    }
522    if len & 15 >= 10 {
523        k2 ^= (bytes[(block_count * read_size) as usize + 9] as u64) << 8;
524    }
525    if len & 15 >= 9 {
526        k2 ^= bytes[(block_count * read_size) as usize + 8] as u64;
527        k2 = k2.wrapping_mul(c2);
528        k2 = k2.rotate_left(33);
529        k2 = k2.wrapping_mul(c1);
530        h2 ^= k2;
531    }
532
533    if len & 15 >= 8 {
534        k1 ^= (bytes[(block_count * read_size) as usize + 7] as u64) << 56;
535    }
536    if len & 15 >= 7 {
537        k1 ^= (bytes[(block_count * read_size) as usize + 6] as u64) << 48;
538    }
539    if len & 15 >= 6 {
540        k1 ^= (bytes[(block_count * read_size) as usize + 5] as u64) << 40;
541    }
542    if len & 15 >= 5 {
543        k1 ^= (bytes[(block_count * read_size) as usize + 4] as u64) << 32;
544    }
545    if len & 15 >= 4 {
546        k1 ^= (bytes[(block_count * read_size) as usize + 3] as u64) << 24;
547    }
548    if len & 15 >= 3 {
549        k1 ^= (bytes[(block_count * read_size) as usize + 2] as u64) << 16;
550    }
551    if len & 15 >= 2 {
552        k1 ^= (bytes[(block_count * read_size) as usize + 1] as u64) << 8;
553    }
554    if len & 15 >= 1 {
555        k1 ^= bytes[(block_count * read_size) as usize] as u64;
556        k1 = k1.wrapping_mul(c1);
557        k1 = k1.rotate_left(31);
558        k1 = k1.wrapping_mul(c2);
559        h1 ^= k1;
560    }
561
562    h1 ^= bytes.len() as u64;
563    h2 ^= bytes.len() as u64;
564
565    h1 = h1.wrapping_add(h2);
566    h2 = h2.wrapping_add(h1);
567
568    h1 = fmix64(h1);
569    h2 = fmix64(h2);
570
571    h1 = h1.wrapping_add(h2);
572    h2 = h2.wrapping_add(h1);
573
574    (h1, h2)
575}
576
577#[cfg(test)]
578mod test_hash {
579    use super::murmurhash3_x64_128;
580
581    #[test]
582    fn test_empty_string() {
583        assert!(murmurhash3_x64_128("".as_bytes(), 0) == (0, 0));
584    }
585
586    #[test]
587    fn test_tail_lengths() {
588        assert!(
589            murmurhash3_x64_128("1".as_bytes(), 0) == (8213365047359667313, 10676604921780958775)
590        );
591        assert!(
592            murmurhash3_x64_128("12".as_bytes(), 0) == (5355690773644049813, 9855895140584599837)
593        );
594        assert!(
595            murmurhash3_x64_128("123".as_bytes(), 0) == (10978418110857903978, 4791445053355511657)
596        );
597        assert!(
598            murmurhash3_x64_128("1234".as_bytes(), 0) == (619023178690193332, 3755592904005385637)
599        );
600        assert!(
601            murmurhash3_x64_128("12345".as_bytes(), 0)
602                == (2375712675693977547, 17382870096830835188)
603        );
604        assert!(
605            murmurhash3_x64_128("123456".as_bytes(), 0)
606                == (16435832985690558678, 5882968373513761278)
607        );
608        assert!(
609            murmurhash3_x64_128("1234567".as_bytes(), 0)
610                == (3232113351312417698, 4025181827808483669)
611        );
612        assert!(
613            murmurhash3_x64_128("12345678".as_bytes(), 0)
614                == (4272337174398058908, 10464973996478965079)
615        );
616        assert!(
617            murmurhash3_x64_128("123456789".as_bytes(), 0)
618                == (4360720697772133540, 11094893415607738629)
619        );
620        assert!(
621            murmurhash3_x64_128("123456789a".as_bytes(), 0)
622                == (12594836289594257748, 2662019112679848245)
623        );
624        assert!(
625            murmurhash3_x64_128("123456789ab".as_bytes(), 0)
626                == (6978636991469537545, 12243090730442643750)
627        );
628        assert!(
629            murmurhash3_x64_128("123456789abc".as_bytes(), 0)
630                == (211890993682310078, 16480638721813329343)
631        );
632        assert!(
633            murmurhash3_x64_128("123456789abcd".as_bytes(), 0)
634                == (12459781455342427559, 3193214493011213179)
635        );
636        assert!(
637            murmurhash3_x64_128("123456789abcde".as_bytes(), 0)
638                == (12538342858731408721, 9820739847336455216)
639        );
640        assert!(
641            murmurhash3_x64_128("123456789abcdef".as_bytes(), 0)
642                == (9165946068217512774, 2451472574052603025)
643        );
644        assert!(
645            murmurhash3_x64_128("123456789abcdef1".as_bytes(), 0)
646                == (9259082041050667785, 12459473952842597282)
647        );
648    }
649
650    #[test]
651    fn test_large_data() {
652        assert!(murmurhash3_x64_128("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam at consequat massa. Cras eleifend pellentesque ex, at dignissim libero maximus ut. Sed eget nulla felis".as_bytes(), 0)
653            == (9455322759164802692, 17863277201603478371));
654    }
655
656    #[test]
657    fn test_unaligned_full_block() {
658        let data = b"x123456789abcdef1";
659        assert!(murmurhash3_x64_128(&data[1..], 0) == (9259082041050667785, 12459473952842597282));
660    }
661}
662
663#[cfg(test)]
664mod case_tests {
665    use super::*;
666
667    #[test]
668    fn test_to_snake_case() {
669        assert_eq!(to_snake_case("camelCase"), "camel_case");
670        assert_eq!(to_snake_case("PascalCase"), "pascal_case");
671        assert_eq!(to_snake_case("HTTPRequest"), "http_request");
672        assert_eq!(to_snake_case("simpleTest"), "simple_test");
673        assert_eq!(to_snake_case("already_snake"), "already_snake");
674        assert_eq!(to_snake_case("ABC"), "abc");
675    }
676
677    #[test]
678    fn test_to_camel_case() {
679        assert_eq!(to_camel_case("snake_case"), "snakeCase");
680        assert_eq!(to_camel_case("simple_test"), "simpleTest");
681        assert_eq!(to_camel_case("already"), "already");
682        assert_eq!(to_camel_case("a_b_c"), "aBC");
683    }
684}
685
686pub mod buffer_rw_string {
687    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
688    use std::arch::aarch64::*;
689    #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
690    use std::arch::x86_64::*;
691    #[cfg(all(
692        any(target_arch = "x86", target_arch = "x86_64"),
693        target_feature = "sse2",
694        not(target_feature = "avx2")
695    ))]
696    use std::arch::x86_64::*;
697
698    use crate::buffer::{Reader, Writer};
699    use crate::error::Error;
700
701    #[inline(always)]
702    fn latin1_utf8_capacity(len: usize) -> Result<usize, Error> {
703        len.checked_mul(2)
704            .filter(|&capacity| capacity <= isize::MAX as usize)
705            .ok_or_else(|| Error::invalid_data("Latin1 string exceeds native allocation limit"))
706    }
707
708    #[inline]
709    pub fn write_latin1_standard(writer: &mut Writer, s: &str) {
710        for c in s.chars() {
711            let b = c as u32;
712            assert!(b <= 0xFF, "Non-Latin1 character found");
713            writer.write_u8(b as u8);
714        }
715    }
716
717    #[inline(always)]
718    pub fn write_latin1_string(writer: &mut Writer, s: &str) {
719        if s.len() < 128 {
720            // Fast path for small buffers
721            let bytes = s.as_bytes();
722            // CRITICAL: Only safe if ASCII (UTF-8 == Latin1 for ASCII)
723            let is_ascii = bytes.iter().all(|&b| b < 0x80);
724            if is_ascii {
725                writer.bf.reserve(s.len());
726                writer.bf.extend_from_slice(bytes);
727            } else {
728                // Non-ASCII: must iterate chars to extract Latin1 byte values
729                writer.bf.reserve(s.len());
730                for c in s.chars() {
731                    let v = c as u32;
732                    assert!(v <= 0xFF, "Non-Latin1 character found");
733                    writer.bf.push(v as u8);
734                }
735            }
736            return;
737        }
738        write_latin1_simd(writer, s);
739    }
740
741    #[inline]
742    pub fn write_utf8_standard(writer: &mut Writer, s: &str) {
743        let bytes = s.as_bytes();
744        writer.bf.extend_from_slice(bytes);
745    }
746
747    #[inline]
748    pub fn write_utf16_standard(writer: &mut Writer, utf16: &[u16]) {
749        #[cfg(target_endian = "little")]
750        {
751            let total_bytes = utf16.len() * 2;
752            let old_len = writer.bf.len();
753            writer.bf.reserve(total_bytes);
754            unsafe {
755                let dest = writer.bf.as_mut_ptr().add(old_len);
756                let src = utf16.as_ptr() as *const u8;
757                std::ptr::copy_nonoverlapping(src, dest, total_bytes);
758                writer.bf.set_len(old_len + total_bytes);
759            }
760        }
761        #[cfg(target_endian = "big")]
762        {
763            let total_bytes = utf16.len() * 2;
764            let old_len = writer.bf.len();
765            writer.bf.reserve(total_bytes);
766            unsafe {
767                let dest = writer.bf.as_mut_ptr().add(old_len);
768                // Need to swap bytes for each u16 to little-endian
769                for (i, &unit) in utf16.iter().enumerate() {
770                    let swapped = unit.swap_bytes();
771                    let ptr = dest.add(i * 2) as *mut u16;
772                    std::ptr::write_unaligned(ptr, swapped);
773                }
774                writer.bf.set_len(old_len + total_bytes);
775            }
776        }
777    }
778
779    #[inline]
780    pub fn read_latin1_standard(reader: &mut Reader, len: usize) -> Result<String, Error> {
781        let slice = reader.sub_slice(reader.get_cursor(), reader.get_cursor() + len)?;
782        let result: String = slice.iter().map(|&b| b as char).collect();
783        reader.move_next(len);
784        Ok(result)
785    }
786
787    #[inline]
788    pub fn read_utf8_standard(reader: &mut Reader, len: usize) -> Result<String, Error> {
789        let slice = reader.sub_slice(reader.get_cursor(), reader.get_cursor() + len)?;
790        // Rust is the only runtime that checks UTF-8 string payloads by default; borrow first so
791        // the check adds no temporary Vec before constructing the final String.
792        let value = std::str::from_utf8(slice)
793            .map_err(|_| Error::encoding_error("invalid UTF-8 string"))?
794            .to_owned();
795        reader.move_next(len);
796        Ok(value)
797    }
798
799    #[inline]
800    pub fn read_utf16_standard(reader: &mut Reader, len: usize) -> Result<String, Error> {
801        if len % 2 != 0 {
802            return Err(Error::encoding_error("UTF-16 length must be even"));
803        }
804        let bytes = reader.read_bytes(len)?;
805        let units: Vec<u16> = bytes
806            .chunks_exact(2)
807            .map(|c| u16::from_le_bytes([c[0], c[1]]))
808            .collect();
809        Ok(String::from_utf16_lossy(&units))
810    }
811
812    #[inline]
813    fn is_ascii_bytes(bytes: &[u8]) -> bool {
814        let len = bytes.len();
815        let mut i = 0;
816
817        #[cfg(target_arch = "x86_64")]
818        unsafe {
819            if is_x86_feature_detected!("avx2") && len >= 32 {
820                while i + 32 <= len {
821                    let chunk = _mm256_loadu_si256(bytes.as_ptr().add(i) as *const __m256i);
822                    let mask = _mm256_movemask_epi8(chunk);
823                    if mask != 0 {
824                        return false;
825                    }
826                    i += 32;
827                }
828            }
829        }
830
831        #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
832        unsafe {
833            if is_x86_feature_detected!("sse2") && len >= 16 {
834                while i + 16 <= len {
835                    let chunk = _mm_loadu_si128(bytes.as_ptr().add(i) as *const __m128i);
836                    let mask = _mm_movemask_epi8(chunk);
837                    if mask != 0 {
838                        return false;
839                    }
840                    i += 16;
841                }
842            }
843        }
844
845        #[cfg(target_arch = "aarch64")]
846        unsafe {
847            if std::arch::is_aarch64_feature_detected!("neon") && len >= 16 {
848                while i + 16 <= len {
849                    let chunk = vld1q_u8(bytes.as_ptr().add(i));
850                    if vmaxvq_u8(chunk) >= 0x80 {
851                        return false;
852                    }
853                    i += 16;
854                }
855            }
856        }
857
858        // Scalar fallback
859        bytes[i..].iter().all(|&b| b < 0x80)
860    }
861
862    #[inline]
863    pub fn write_latin1_simd(writer: &mut Writer, s: &str) {
864        if s.is_empty() {
865            return;
866        }
867
868        let bytes = s.as_bytes();
869
870        // CRITICAL OPTIMIZATION: For ASCII strings, UTF-8 bytes == Latin1 bytes
871        // Check if all ASCII using SIMD
872        if is_ascii_bytes(bytes) {
873            // Zero-copy fast path: direct write
874            let len = bytes.len();
875            writer.bf.reserve(len);
876            writer.bf.extend_from_slice(bytes);
877        } else {
878            // Non-ASCII: Must iterate chars to extract Latin1 byte values
879            // Example: 'À' in Rust String is UTF-8 [0xC3, 0x80] but Latin1 is [0xC0]
880            let mut buf: Vec<u8> = Vec::with_capacity(s.len());
881            for c in s.chars() {
882                let v = c as u32;
883                assert!(v <= 0xFF, "Non-Latin1 character found");
884                buf.push(v as u8);
885            }
886            let len = buf.len();
887            writer.bf.reserve(len);
888            writer.bf.extend_from_slice(&buf);
889        }
890    }
891
892    #[inline]
893    pub fn read_latin1_simd(reader: &mut Reader, len: usize) -> Result<String, Error> {
894        if len == 0 {
895            return Ok(String::new());
896        }
897        // Pessimistic allocation: Latin1 0x80-0xFF expands to 2 bytes in UTF-8.
898        let capacity = latin1_utf8_capacity(len)?;
899        let src = reader.read_bytes(len)?;
900        let mut out: Vec<u8> = Vec::with_capacity(capacity);
901
902        unsafe {
903            let out_ptr = out.as_mut_ptr();
904            let mut out_len = 0usize;
905            let mut i = 0usize;
906
907            // ---- AVX2 fast-path: process 32 ASCII bytes at once ----
908            #[cfg(target_arch = "x86_64")]
909            {
910                if std::arch::is_x86_feature_detected!("avx2") {
911                    use std::arch::x86_64::*;
912                    while i + 32 <= len {
913                        let ptr = src.as_ptr().add(i) as *const __m256i;
914                        let chunk = _mm256_loadu_si256(ptr);
915                        let mask = _mm256_movemask_epi8(chunk);
916                        if mask == 0 {
917                            // All ASCII: direct copy (no conversion needed)
918                            _mm256_storeu_si256(out_ptr.add(out_len) as *mut __m256i, chunk);
919                            out_len += 32;
920                            i += 32;
921                            continue;
922                        } else {
923                            // Contains Latin1 bytes, break to scalar
924                            break;
925                        }
926                    }
927                }
928            }
929
930            // ---- SSE2 fast-path: process 16 ASCII bytes at once ----
931            #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
932            {
933                if std::arch::is_x86_feature_detected!("sse2") {
934                    use std::arch::x86_64::*;
935                    while i + 16 <= len {
936                        let ptr = src.as_ptr().add(i) as *const __m128i;
937                        let chunk = _mm_loadu_si128(ptr);
938                        let mask = _mm_movemask_epi8(chunk);
939                        if mask == 0 {
940                            // All ASCII: direct copy
941                            _mm_storeu_si128(out_ptr.add(out_len) as *mut __m128i, chunk);
942                            out_len += 16;
943                            i += 16;
944                            continue;
945                        } else {
946                            break;
947                        }
948                    }
949                }
950            }
951
952            // ---- NEON fast-path: process 16 ASCII bytes at once ----
953            #[cfg(target_arch = "aarch64")]
954            {
955                if std::arch::is_aarch64_feature_detected!("neon") {
956                    use std::arch::aarch64::*;
957                    while i + 16 <= len {
958                        let ptr = src.as_ptr().add(i);
959                        let v = vld1q_u8(ptr);
960                        // Check if any byte >= 0x80
961                        if vmaxvq_u8(v) < 0x80 {
962                            // All ASCII: direct copy
963                            vst1q_u8(out_ptr.add(out_len), v);
964                            out_len += 16;
965                            i += 16;
966                            continue;
967                        } else {
968                            break;
969                        }
970                    }
971                }
972            }
973
974            // ---- Scalar fallback: convert Latin1 -> UTF-8 ----
975            // ASCII (0x00-0x7F): copy as-is
976            // Latin1 (0x80-0xFF): encode as 2-byte UTF-8
977            while i < len {
978                let b = *src.get_unchecked(i);
979                if b < 0x80 {
980                    *out_ptr.add(out_len) = b;
981                    out_len += 1;
982                } else {
983                    // Latin1 byte 0x80-0xFF -> UTF-8 encoding
984                    // Example: 0xC0 (À) -> [0xC3, 0x80]
985                    *out_ptr.add(out_len) = 0xC0 | (b >> 6);
986                    *out_ptr.add(out_len + 1) = 0x80 | (b & 0x3F);
987                    out_len += 2;
988                }
989                i += 1;
990            }
991
992            out.set_len(out_len);
993        }
994        Ok(unsafe { String::from_utf8_unchecked(out) })
995    }
996
997    #[cfg(test)]
998    mod tests {
999        use super::*;
1000        use crate::buffer::{Reader, Writer};
1001
1002        #[test]
1003        fn test_latin1() {
1004            let samples = [
1005                "Hello World!",
1006                "Rusty Café",
1007                "1234567890",
1008                "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝ",
1009            ];
1010
1011            for s in samples {
1012                let mut buffer = vec![];
1013                let mut writer = Writer::from_buffer(&mut buffer);
1014                write_latin1_simd(&mut writer, s);
1015                write_latin1_simd(&mut writer, s);
1016                let bytes = &*writer.dump();
1017                let bytes_len = bytes.len() / 2;
1018                let mut reader = Reader::new(bytes);
1019                assert_eq!(read_latin1_standard(&mut reader, bytes_len).unwrap(), s);
1020                assert_eq!(read_latin1_standard(&mut reader, bytes_len).unwrap(), s);
1021
1022                let mut buffer = vec![];
1023                let mut writer = Writer::from_buffer(&mut buffer);
1024                write_latin1_standard(&mut writer, s);
1025                write_latin1_standard(&mut writer, s);
1026                let bytes = &*writer.dump();
1027                let bytes_len = bytes.len() / 2;
1028                let mut reader = Reader::new(bytes);
1029                assert_eq!(read_latin1_simd(&mut reader, bytes_len).unwrap(), s);
1030                assert_eq!(read_latin1_simd(&mut reader, bytes_len).unwrap(), s);
1031            }
1032        }
1033
1034        #[test]
1035        fn test_utf8() {
1036            let samples = [
1037                "hello",
1038                "rust语言",
1039                "你好，世界",
1040                "emoji 😀😃😄😁",
1041                "mixed ASCII + 中文 + emoji 😁",
1042            ];
1043
1044            for s in samples {
1045                let bytes_len = s.len();
1046
1047                let mut buffer = vec![];
1048                let mut writer = Writer::from_buffer(&mut buffer);
1049                write_utf8_standard(&mut writer, s);
1050                write_utf8_standard(&mut writer, s);
1051                let bytes = &*writer.dump();
1052                let mut reader = Reader::new(bytes);
1053                assert_eq!(read_utf8_standard(&mut reader, bytes_len).unwrap(), s);
1054                assert_eq!(read_utf8_standard(&mut reader, bytes_len).unwrap(), s);
1055            }
1056        }
1057
1058        #[test]
1059        fn test_utf16() {
1060            let samples = [
1061                "hello",
1062                "rust语言",
1063                "你好，世界",
1064                "emoji 😀😃😄😁",
1065                "混合文字 + emoji 🐍💻🦀",
1066            ];
1067            for s in samples {
1068                let utf16: Vec<u16> = s.encode_utf16().collect();
1069                let bytes_len = utf16.len() * 2;
1070
1071                let mut buffer = vec![];
1072                let mut writer = Writer::from_buffer(&mut buffer);
1073                write_utf16_standard(&mut writer, &utf16);
1074                write_utf16_standard(&mut writer, &utf16);
1075
1076                let mut buffer = vec![];
1077                let mut writer = Writer::from_buffer(&mut buffer);
1078                write_utf16_standard(&mut writer, &utf16);
1079                write_utf16_standard(&mut writer, &utf16);
1080                let bytes = &*writer.dump();
1081                let mut reader = Reader::new(bytes);
1082                assert_eq!(read_utf16_standard(&mut reader, bytes_len).unwrap(), s);
1083                assert_eq!(read_utf16_standard(&mut reader, bytes_len).unwrap(), s);
1084            }
1085        }
1086
1087        #[test]
1088        fn utf16_truncated_read() {
1089            let mut reader = Reader::new(&[0, 1]);
1090
1091            assert!(read_utf16_standard(&mut reader, 4).is_err());
1092        }
1093
1094        #[test]
1095        fn utf16_length_overflow() {
1096            let mut reader = Reader::new(&[0, 1]);
1097            reader.skip(2).unwrap();
1098
1099            assert!(read_utf16_standard(&mut reader, usize::MAX - 1).is_err());
1100        }
1101
1102        #[test]
1103        fn latin1_native_capacity() {
1104            let len = (isize::MAX as usize / 2) + 1;
1105            assert!(latin1_utf8_capacity(len).is_err());
1106        }
1107    }
1108}
fory_core/util/string_util.rs

fory_core/util/
string_util.rs
