punycode_rs/
lib.rs

1//! A Rust port of Punycode (RFC 3492), the ASCII-compatible encoding at the heart of
2//! Internationalized Domain Names — e.g. `münchen` <-> `mnchen-3ya`.
3//!
4//! Port target: Python's standard library `encodings/punycode.py` (Martin v. Löwis).
5//! Verified against Python's built-in `punycode` codec.
6//!
7//! The algorithm separates a string into its ASCII "base" and its sorted non-ASCII
8//! characters, then encodes each non-ASCII character as a single integer "delta" that
9//! records both *which* character and *where* it is inserted. Deltas are written as
10//! base-36 generalized variable-length integers with an adaptive bias that self-tunes
11//! for compactness. Decoding mirrors the process exactly.
12
13use std::fmt;
14
15const BASE: i64 = 36;
16const TMIN: i64 = 1;
17const TMAX: i64 = 26;
18const INITIAL_BIAS: i64 = 72;
19const DAMP: i64 = 700;
20const SKEW: i64 = 38;
21
22/// `a-z0-9` — the 36 base-36 digit characters.
23const DIGITS: &[u8; 36] = b"abcdefghijklmnopqrstuvwxyz0123456789";
24
25/// An error from [`decode`].
26#[derive(Debug, Clone, PartialEq, Eq)]
27pub enum PunycodeError {
28    /// The input contained a non-ASCII byte (Punycode is ASCII-only).
29    NotAscii,
30    /// The extended part ended mid-number.
31    Incomplete,
32    /// The extended part contained a byte that is not a valid base-36 digit.
33    InvalidDigit,
34    /// Decoding produced a code point that is not a valid character.
35    InvalidCodePoint,
36}
37
38impl fmt::Display for PunycodeError {
39    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40        let msg = match self {
41            PunycodeError::NotAscii => "input is not ASCII",
42            PunycodeError::Incomplete => "incomplete punycode string",
43            PunycodeError::InvalidDigit => "invalid extended code point",
44            PunycodeError::InvalidCodePoint => "invalid decoded character",
45        };
46        f.write_str(msg)
47    }
48}
49
50impl std::error::Error for PunycodeError {}
51
52// --- Shared: threshold and bias adaptation. ---
53
54/// `T(j, bias)` — the digit threshold, clamped to `[tmin, tmax]`.
55fn threshold(j: i64, bias: i64) -> i64 {
56    (BASE * (j + 1) - bias).clamp(TMIN, TMAX)
57}
58
59/// Bias adaptation after encoding/decoding a delta. Port of `adapt`.
60fn adapt(mut delta: i64, first: bool, numchars: i64) -> i64 {
61    delta /= if first { DAMP } else { 2 };
62    delta += delta / numchars;
63    let mut divisions = 0;
64    while delta > ((BASE - TMIN) * TMAX) / 2 {
65        delta /= BASE - TMIN;
66        divisions += BASE;
67    }
68    divisions + (BASE * delta) / (delta + SKEW)
69}
70
71// --- Encoding (port of `punycode_encode`). ---
72
73/// 3.1 Basic code point segregation: the ASCII bytes, and the sorted unique non-ASCII chars.
74fn segregate(text: &[char]) -> (Vec<u8>, Vec<char>) {
75    let mut base = Vec::new();
76    let mut extended = std::collections::BTreeSet::new();
77    for &c in text {
78        if (c as u32) < 128 {
79            base.push(c as u8);
80        } else {
81            extended.insert(c);
82        }
83    }
84    (base, extended.into_iter().collect())
85}
86
87/// Count characters in `text` whose code point is below `max`.
88fn selective_len(text: &[char], max: u32) -> i64 {
89    text.iter().filter(|&&c| (c as u32) < max).count() as i64
90}
91
92/// Next occurrence of `target` in `text` after `pos`, returning `(index, pos)` where `index`
93/// counts only characters at or below `target`. `(-1, -1)` when there is no further match.
94fn selective_find(text: &[char], target: char, mut index: i64, mut pos: i64) -> (i64, i64) {
95    let len = text.len() as i64;
96    loop {
97        pos += 1;
98        if pos == len {
99            return (-1, -1);
100        }
101        let c = text[pos as usize];
102        if c == target {
103            return (index + 1, pos);
104        } else if c < target {
105            index += 1;
106        }
107    }
108}
109
110/// 3.2 Insertion unsort coding: the sequence of deltas describing the non-ASCII insertions.
111fn insertion_unsort(text: &[char], extended: &[char]) -> Vec<i64> {
112    let mut oldchar: i64 = 0x80;
113    let mut oldindex: i64 = -1;
114    let mut result = Vec::new();
115
116    for &c in extended {
117        let mut index: i64 = -1;
118        let mut pos: i64 = -1;
119        let ch = c as i64;
120        let curlen = selective_len(text, c as u32);
121        let mut delta = (curlen + 1) * (ch - oldchar);
122        loop {
123            let (i, p) = selective_find(text, c, index, pos);
124            index = i;
125            pos = p;
126            if index == -1 {
127                break;
128            }
129            delta += index - oldindex;
130            result.push(delta - 1);
131            oldindex = index;
132            delta = 0;
133        }
134        oldchar = ch;
135    }
136    result
137}
138
139/// 3.3 Generalized variable-length integer for one delta `n`.
140fn generate_generalized_integer(mut n: i64, bias: i64) -> Vec<u8> {
141    let mut result = Vec::new();
142    let mut j = 0;
143    loop {
144        let t = threshold(j, bias);
145        if n < t {
146            result.push(DIGITS[n as usize]);
147            return result;
148        }
149        result.push(DIGITS[(t + (n - t) % (BASE - t)) as usize]);
150        n = (n - t) / (BASE - t);
151        j += 1;
152    }
153}
154
155/// 3.4 Encode all deltas, adapting the bias between each.
156fn generate_integers(baselen: i64, deltas: &[i64]) -> Vec<u8> {
157    let mut result = Vec::new();
158    let mut bias = INITIAL_BIAS;
159    for (points, &delta) in deltas.iter().enumerate() {
160        result.extend(generate_generalized_integer(delta, bias));
161        bias = adapt(delta, points == 0, baselen + points as i64 + 1);
162    }
163    result
164}
165
166/// Encode a string to Punycode (RFC 3492). Pure-ASCII input gains a trailing `-`.
167pub fn encode(text: &str) -> String {
168    let chars: Vec<char> = text.chars().collect();
169    let (mut base, extended) = segregate(&chars);
170    let deltas = insertion_unsort(&chars, &extended);
171    let encoded = generate_integers(base.len() as i64, &deltas);
172
173    if !base.is_empty() {
174        base.push(b'-');
175    }
176    base.extend(encoded);
177    String::from_utf8(base).expect("punycode output is ASCII")
178}
179
180// --- Decoding (port of `punycode_decode`). ---
181
182/// 3.3 Decode one generalized integer, returning the new position and the value.
183fn decode_generalized_number(
184    extended: &[u8],
185    mut extpos: usize,
186    bias: i64,
187) -> Result<(usize, i64), PunycodeError> {
188    let mut result = 0;
189    let mut w = 1;
190    let mut j = 0;
191    loop {
192        let ch = *extended.get(extpos).ok_or(PunycodeError::Incomplete)?;
193        extpos += 1;
194        let digit = if ch.is_ascii_uppercase() {
195            (ch - b'A') as i64
196        } else if ch.is_ascii_digit() {
197            (ch - b'0') as i64 + 26
198        } else {
199            return Err(PunycodeError::InvalidDigit);
200        };
201        let t = threshold(j, bias);
202        result += digit * w;
203        if digit < t {
204            return Ok((extpos, result));
205        }
206        w *= BASE - t;
207        j += 1;
208    }
209}
210
211/// 3.2 Rebuild the string by decoding deltas and inserting characters. Port of `insertion_sort`.
212fn insertion_sort(mut base: Vec<char>, extended: &[u8]) -> Result<Vec<char>, PunycodeError> {
213    let mut char_code: i64 = 0x80;
214    let mut pos: i64 = -1;
215    let mut bias = INITIAL_BIAS;
216    let mut extpos = 0;
217
218    while extpos < extended.len() {
219        let (newpos, delta) = decode_generalized_number(extended, extpos, bias)?;
220        pos += delta + 1;
221        char_code += pos / (base.len() as i64 + 1);
222        if char_code > 0x10FFFF {
223            return Err(PunycodeError::InvalidCodePoint);
224        }
225        pos %= base.len() as i64 + 1;
226        let ch = char::from_u32(char_code as u32).ok_or(PunycodeError::InvalidCodePoint)?;
227        base.insert(pos as usize, ch);
228        bias = adapt(delta, extpos == 0, base.len() as i64);
229        extpos = newpos;
230    }
231    Ok(base)
232}
233
234/// Decode a Punycode string back to Unicode. Input must be ASCII.
235pub fn decode(text: &str) -> Result<String, PunycodeError> {
236    if !text.is_ascii() {
237        return Err(PunycodeError::NotAscii);
238    }
239    let bytes = text.as_bytes();
240
241    // Split at the last '-': everything before is verbatim ASCII base, after is the
242    // (case-insensitive) extended part.
243    let (base, extended) = match bytes.iter().rposition(|&b| b == b'-') {
244        Some(pos) => (
245            bytes[..pos].iter().map(|&b| b as char).collect(),
246            bytes[pos + 1..].to_ascii_uppercase(),
247        ),
248        None => (Vec::new(), bytes.to_ascii_uppercase()),
249    };
250
251    Ok(insertion_sort(base, &extended)?.into_iter().collect())
252}
253
254#[cfg(test)]
255mod tests {
256    use super::*;
257
258    #[test]
259    fn encodes_known_values() {
260        assert_eq!(encode("münchen"), "mnchen-3ya");
261        assert_eq!(encode("abc"), "abc-"); // pure ASCII gains a trailing dash
262        assert_eq!(encode(""), "");
263        assert_eq!(encode("ü"), "tda");
264    }
265
266    #[test]
267    fn decodes_known_values() {
268        assert_eq!(decode("mnchen-3ya").unwrap(), "münchen");
269        assert_eq!(decode("abc-").unwrap(), "abc");
270        assert_eq!(decode("").unwrap(), "");
271        assert_eq!(decode("tda").unwrap(), "ü");
272    }
273
274    #[test]
275    fn round_trips() {
276        for s in [
277            "münchen",
278            "café",
279            "naïve",
280            "日本語",
281            "Ελληνικά",
282            "abc",
283            "Hello-World",
284            "",
285            "ñ",
286            "a1b2c3",
287            "emoji-💡-here",
288            "mixed café 日本 test",
289        ] {
290            let encoded = encode(s);
291            assert_eq!(decode(&encoded).unwrap(), s, "round-trip failed for {s:?}");
292        }
293    }
294
295    #[test]
296    fn decode_rejects_bad_input() {
297        assert_eq!(decode("café"), Err(PunycodeError::NotAscii)); // non-ASCII input
298        assert!(decode("-!").is_err()); // '!' is not a base-36 digit
299    }
300}
punycode_rs/lib.rs

punycode_rs/
lib.rs
