perl_test_generators/
unicode.rs

1//! Generators for Unicode strings exercising various edge cases.
2//!
3//! Produces strings with BMP characters, supplementary planes, surrogate
4//! boundaries, combining marks, and RTL scripts — useful for testing
5//! UTF-8/UTF-16 position mappers and LSP protocol encoding.
6
7use proptest::prelude::*;
8
9/// Generate a Unicode string suitable for testing UTF-8/UTF-16 conversion.
10///
11/// Mixes ASCII, BMP non-ASCII, supplementary plane characters, and
12/// combining marks.
13pub fn unicode_string() -> impl Strategy<Value = String> {
14    prop_oneof![
15        // Pure ASCII
16        prop::collection::vec(
17            prop_oneof![
18                prop::char::range('a', 'z'),
19                prop::char::range('A', 'Z'),
20                prop::char::range('0', '9'),
21                Just(' '),
22                Just('\t'),
23            ],
24            0..=50_usize,
25        )
26        .prop_map(|chars| chars.into_iter().collect()),
27        // BMP with non-ASCII (Latin-1 supplement, CJK, etc.)
28        prop::collection::vec(prop::char::range('\u{00C0}', '\u{FFFF}'), 0..=50_usize)
29            .prop_map(|chars| chars.into_iter().collect()),
30        // Supplementary plane characters (emoji, rare scripts)
31        prop::collection::vec(prop::char::range('\u{10000}', '\u{10FFFF}'), 0..=30_usize)
32            .prop_map(|chars| chars.into_iter().collect()),
33        // Mix of all ranges
34        prop::collection::vec(
35            prop_oneof![
36                prop::char::range('a', 'z'),
37                prop::char::range('A', 'Z'),
38                prop::char::range('0', '9'),
39                prop::char::range('\u{00C0}', '\u{024F}'),
40                prop::char::range('\u{4E00}', '\u{9FFF}'),
41                prop::char::range('\u{10000}', '\u{10FFFF}'),
42                Just('\t'),
43                Just('\n'),
44            ],
45            0..=100_usize,
46        )
47        .prop_map(|chars| chars.into_iter().collect()),
48    ]
49}
50
51/// Generate a non-empty Unicode string.
52///
53/// Useful when call sites need at least one code point and should avoid
54/// separate assumptions/filters in their property tests.
55pub fn non_empty_unicode_string() -> impl Strategy<Value = String> {
56    unicode_string().prop_filter("string must be non-empty", |value| !value.is_empty())
57}
58
59#[cfg(test)]
60mod tests {
61    use super::*;
62
63    proptest! {
64        #[test]
65        fn unicode_string_is_valid_utf8(s in unicode_string()) {
66            // If it compiled as String, it's valid UTF-8. Verify round-trip.
67            let bytes = s.as_bytes();
68            match std::str::from_utf8(bytes) {
69                Ok(roundtrip) => prop_assert_eq!(s.as_str(), roundtrip),
70                Err(err) => prop_assert!(false, "generated string was not valid UTF-8: {err}"),
71            }
72        }
73
74        #[test]
75        fn utf16_len_agrees_with_encode_utf16(s in unicode_string()) {
76            let encoded: Vec<u16> = s.encode_utf16().collect();
77            let from_utf16 = String::from_utf16_lossy(&encoded);
78            prop_assert_eq!(s, from_utf16);
79        }
80
81        #[test]
82        fn non_empty_unicode_string_never_empty(s in non_empty_unicode_string()) {
83            prop_assert!(!s.is_empty(), "non-empty strategy produced an empty string");
84        }
85    }
86}
perl_test_generators/unicode.rs

perl_test_generators/
unicode.rs
