dataprof_core/
text_units.rs

1//! The unit every reported text length is measured in.
2//!
3//! `TextStats::min_length`, `max_length` and `avg_length` count **Unicode
4//! scalar values**, not UTF-8 bytes and not grapheme clusters.
5//!
6//! Bytes were the accidental unit until 0.12: every accumulator reached for
7//! Rust's `str::len()`, so `"東京"` reported a length of 6 and `"🙂"` a length
8//! of 4 under a field named `max_length` (#627). The counts were consistent
9//! across every engine, and consistently surprising for anything but ASCII.
10//!
11//! Scalar values are the least surprising interpretation that costs no
12//! dependency. Grapheme clusters are closer to what a reader would call a
13//! character — `"e\u{0301}"` is one grapheme and two scalars — but segmenting
14//! them correctly needs a Unicode segmentation table and a policy for which
15//! version of it, which is a larger commitment than a profiler's length field
16//! justifies. Encoded size is a property of the encoding rather than the value,
17//! and is available at the source level; it is deliberately not reported under
18//! a name like "length".
19//!
20//! ASCII is unaffected: for ASCII text all three units agree.
21
22/// The unit a report's text lengths were measured in, as recorded in its
23/// metric semantics.
24///
25/// Only the current unit has a value. Releases that counted UTF-8 bytes
26/// recorded no unit at all, and a report from one reads back with the unit
27/// unknown rather than as a byte count it never declared.
28#[derive(
29    Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema,
30)]
31#[serde(rename_all = "snake_case")]
32pub enum TextLengthUnit {
33    /// Unicode scalar values, as counted by [`char_len`].
34    UnicodeScalar,
35}
36
37/// Number of Unicode scalar values in `value`.
38///
39/// This is the single definition of "length" behind every text statistic, so
40/// the CSV, JSON, Parquet, Arrow and in-memory paths cannot drift apart on what
41/// they are counting.
42#[inline]
43pub fn char_len(value: &str) -> usize {
44    // `Chars::count` counts non-continuation bytes rather than decoding each
45    // scalar, so this stays close to `len()` in cost.
46    value.chars().count()
47}
48
49#[cfg(test)]
50mod tests {
51    use super::char_len;
52
53    #[test]
54    fn ascii_is_unchanged() {
55        for value in ["", "a", "hello", "1234567890"] {
56            assert_eq!(char_len(value), value.len(), "{value:?}");
57        }
58    }
59
60    #[test]
61    fn multibyte_scalars_count_once() {
62        // The issue's reproduction: UTF-8 widths are 1, 2, 6, 4.
63        assert_eq!(char_len("a"), 1);
64        assert_eq!(char_len("é"), 1);
65        assert_eq!(char_len("東京"), 2);
66        assert_eq!(char_len("🙂"), 1);
67    }
68
69    #[test]
70    fn a_combining_sequence_counts_each_scalar() {
71        // Two scalars, one grapheme. Documented rather than segmented: the
72        // precomposed and decomposed spellings of the same word differ in
73        // length, which is a property of scalar counting, not a defect.
74        let precomposed = "\u{00e9}";
75        let decomposed = "e\u{0301}";
76        assert_eq!(char_len(precomposed), 1);
77        assert_eq!(char_len(decomposed), 2);
78        // They differ under the old byte unit too, by a different amount, so
79        // neither unit makes the two spellings compare equal.
80        assert_eq!((precomposed.len(), decomposed.len()), (2, 3));
81    }
82
83    #[test]
84    fn an_emoji_sequence_counts_each_scalar() {
85        // A ZWJ family is one grapheme and seven scalars.
86        assert_eq!(char_len("👨‍👩‍👧‍👦"), 7);
87    }
88}
dataprof_core/text_units.rs

dataprof_core/
text_units.rs
