uucore 0.8.0

uutils ~ 'core' uutils code library (cross-platform)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

// This file is part of the uutils coreutils package.
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

// spell-checker:ignore langinfo charmap eucjp euckr euctw CTYPE HKSCS hkscs localedata

//! Locale-aware multi-byte character length detection via `LC_CTYPE`.

use std::sync::OnceLock;

enum MbEncoding {
    Utf8,
    Gb18030,
    EucJp,
    EucKr,
    Big5,
}

fn encoding_from_name(enc: &str) -> MbEncoding {
    match enc {
        "gb18030" | "gbk" | "gb2312" => MbEncoding::Gb18030,
        "euc-jp" | "eucjp" => MbEncoding::EucJp,
        "euc-kr" | "euckr" => MbEncoding::EucKr,
        "big5" | "big5-hkscs" | "big5hkscs" | "euc-tw" | "euctw" => MbEncoding::Big5,
        _ => MbEncoding::Utf8,
    }
}

fn get_encoding() -> &'static MbEncoding {
    static ENCODING: OnceLock<MbEncoding> = OnceLock::new();
    ENCODING.get_or_init(|| {
        let val = ["LC_ALL", "LC_CTYPE", "LANG"]
            .iter()
            .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty()));
        let s = match val.as_deref() {
            Some(s) if s != "C" && s != "POSIX" => s,
            _ => return MbEncoding::Utf8,
        };
        if let Some(enc) = s.split('.').nth(1) {
            let enc = enc.split('@').next().unwrap_or(enc);
            encoding_from_name(&enc.to_ascii_lowercase())
        } else {
            // Bare locale defaults from glibc localedata/SUPPORTED
            match s.split('@').next().unwrap_or(s) {
                "zh_CN" | "zh_SG" => MbEncoding::Gb18030,
                "zh_TW" | "zh_HK" => MbEncoding::Big5,
                _ => MbEncoding::Utf8,
            }
        }
    })
}

/// Byte length of the first character in `bytes` under the current locale encoding.
pub fn mb_char_len(bytes: &[u8]) -> usize {
    debug_assert!(!bytes.is_empty());
    let b0 = bytes[0];
    if b0 <= 0x7F {
        return 1;
    }
    match get_encoding() {
        MbEncoding::Utf8 => utf8_len(bytes, b0),
        MbEncoding::Gb18030 => gb18030_len(bytes, b0),
        MbEncoding::EucJp => eucjp_len(bytes, b0),
        MbEncoding::EucKr => euckr_len(bytes, b0),
        MbEncoding::Big5 => big5_len(bytes, b0),
    }
}

// All helpers below assume b0 > 0x7F (ASCII already handled by caller).

fn utf8_len(b: &[u8], b0: u8) -> usize {
    let n = match b0 {
        0xC2..=0xDF => 2,
        0xE0..=0xEF => 3,
        0xF0..=0xF4 => 4,
        _ => return 1,
    };
    if b.len() >= n && b[1..n].iter().all(|&c| c & 0xC0 == 0x80) {
        n
    } else {
        1
    }
}

// 2-byte: [81-FE][40-7E,80-FE]  4-byte: [81-FE][30-39][81-FE][30-39]
fn gb18030_len(b: &[u8], b0: u8) -> usize {
    if !(0x81..=0xFE).contains(&b0) {
        return 1;
    }
    if b.len() >= 4
        && (0x30..=0x39).contains(&b[1])
        && (0x81..=0xFE).contains(&b[2])
        && (0x30..=0x39).contains(&b[3])
    {
        return 4;
    }
    if b.len() >= 2 && ((0x40..=0x7E).contains(&b[1]) || (0x80..=0xFE).contains(&b[1])) {
        return 2;
    }
    1
}

// 3-byte: [8F][A1-FE][A1-FE]  2-byte: [8E][A1-DF] or [A1-FE][A1-FE]
fn eucjp_len(b: &[u8], b0: u8) -> usize {
    if b0 == 0x8F && b.len() >= 3 && (0xA1..=0xFE).contains(&b[1]) && (0xA1..=0xFE).contains(&b[2])
    {
        return 3;
    }
    if b.len() >= 2 {
        if b0 == 0x8E && (0xA1..=0xDF).contains(&b[1]) {
            return 2;
        }
        if (0xA1..=0xFE).contains(&b0) && (0xA1..=0xFE).contains(&b[1]) {
            return 2;
        }
    }
    1
}

// 2-byte: [A1-FE][A1-FE]
fn euckr_len(b: &[u8], b0: u8) -> usize {
    if (0xA1..=0xFE).contains(&b0) && b.len() >= 2 && (0xA1..=0xFE).contains(&b[1]) {
        2
    } else {
        1
    }
}

// 2-byte: [81-FE][40-7E,A1-FE]
fn big5_len(b: &[u8], b0: u8) -> usize {
    if (0x81..=0xFE).contains(&b0)
        && b.len() >= 2
        && ((0x40..=0x7E).contains(&b[1]) || (0xA1..=0xFE).contains(&b[1]))
    {
        2
    } else {
        1
    }
}