nautilus_core/string/
urlencoding.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! URL percent-encoding and decoding per [RFC 3986].
17//!
18//! The unreserved set is `ALPHA / DIGIT / "-" / "." / "_" / "~"`; every other
19//! byte is percent-encoded as `%HH` using uppercase hexadecimal as recommended
20//! by [RFC 3986 Section 2.1].
21//!
22//! Decoding accepts both uppercase and lowercase hex. A `%` that is not
23//! followed by two hex digits is passed through literally, matching the
24//! behaviour of the `urlencoding` crate that this module replaces.
25//!
26//! [RFC 3986]: https://datatracker.ietf.org/doc/html/rfc3986
27//! [RFC 3986 Section 2.1]: https://datatracker.ietf.org/doc/html/rfc3986#section-2.1
28
29use std::{borrow::Cow, fmt::Display, string::FromUtf8Error};
30
31const UNRESERVED: [bool; 256] = {
32    let mut table = [false; 256];
33    let mut i = b'0';
34    while i <= b'9' {
35        table[i as usize] = true;
36        i += 1;
37    }
38    i = b'A';
39    while i <= b'Z' {
40        table[i as usize] = true;
41        i += 1;
42    }
43    i = b'a';
44    while i <= b'z' {
45        table[i as usize] = true;
46        i += 1;
47    }
48    table[b'-' as usize] = true;
49    table[b'.' as usize] = true;
50    table[b'_' as usize] = true;
51    table[b'~' as usize] = true;
52    table
53};
54
55const ENCODE_PAIR: [[u8; 2]; 256] = {
56    const NIBBLE: [u8; 16] = *b"0123456789ABCDEF";
57    let mut table = [[0u8; 2]; 256];
58    let mut i = 0u16;
59    while i < 256 {
60        table[i as usize] = [NIBBLE[(i >> 4) as usize], NIBBLE[(i & 0x0f) as usize]];
61        i += 1;
62    }
63    table
64};
65
66// 0xFF sentinel marks non-hex characters
67const DECODE_NIBBLE: [u8; 256] = {
68    let mut table = [0xFFu8; 256];
69    let mut i = 0u8;
70    while i < 10 {
71        table[(b'0' + i) as usize] = i;
72        i += 1;
73    }
74    i = 0;
75    while i < 6 {
76        table[(b'a' + i) as usize] = 10 + i;
77        table[(b'A' + i) as usize] = 10 + i;
78        i += 1;
79    }
80    table
81};
82
83/// Percent-encodes a string per RFC 3986.
84///
85/// Returns the input borrowed when every byte is already in the unreserved
86/// set, otherwise an owned encoded copy.
87///
88/// # Panics
89///
90/// Never panics in practice: [`encode_bytes`] only emits ASCII bytes
91/// (unreserved characters or `%HH` pairs), so [`String::from_utf8`] always
92/// succeeds.
93#[must_use]
94pub fn encode(input: &str) -> Cow<'_, str> {
95    match encode_bytes(input.as_bytes()) {
96        Cow::Borrowed(_) => Cow::Borrowed(input),
97        Cow::Owned(bytes) => Cow::Owned(String::from_utf8(bytes).expect("encoded output is ASCII")),
98    }
99}
100
101/// Percent-encodes a byte slice per RFC 3986.
102///
103/// Returns the input borrowed when every byte is already in the unreserved
104/// set, otherwise an owned encoded copy.
105#[must_use]
106pub fn encode_bytes(input: &[u8]) -> Cow<'_, [u8]> {
107    let Some(first) = input.iter().position(|&b| !UNRESERVED[b as usize]) else {
108        return Cow::Borrowed(input);
109    };
110
111    // Slack for payloads dominated by reserved chars without over-allocating
112    // on mostly-unreserved inputs; Vec's geometric growth covers the rest.
113    let mut out = Vec::with_capacity(input.len() + input.len() / 2 + 16);
114    out.extend_from_slice(&input[..first]);
115
116    let mut rest = &input[first..];
117    while let Some(&byte) = rest.first() {
118        if UNRESERVED[byte as usize] {
119            let run_end = rest
120                .iter()
121                .position(|&b| !UNRESERVED[b as usize])
122                .unwrap_or(rest.len());
123            out.extend_from_slice(&rest[..run_end]);
124            rest = &rest[run_end..];
125        } else {
126            out.push(b'%');
127            out.extend_from_slice(&ENCODE_PAIR[byte as usize]);
128            rest = &rest[1..];
129        }
130    }
131    Cow::Owned(out)
132}
133
134/// Percent-decodes a string per RFC 3986.
135///
136/// Returns the input borrowed when no `%` is present. Otherwise decodes
137/// `%HH` pairs (hex is case-insensitive) and leaves any `%` that is not
138/// followed by two hex digits in place.
139///
140/// # Errors
141///
142/// Returns [`DecodeError::InvalidUtf8`] if the decoded bytes are not valid
143/// UTF-8.
144pub fn decode(input: &str) -> Result<Cow<'_, str>, DecodeError> {
145    match decode_bytes(input.as_bytes()) {
146        Cow::Borrowed(_) => Ok(Cow::Borrowed(input)),
147        Cow::Owned(bytes) => String::from_utf8(bytes)
148            .map(Cow::Owned)
149            .map_err(DecodeError::InvalidUtf8),
150    }
151}
152
153/// Percent-decodes a byte slice.
154///
155/// Returns the input borrowed when no `%` is present. A `%` that is not
156/// followed by two hex digits is left in place.
157#[must_use]
158pub fn decode_bytes(input: &[u8]) -> Cow<'_, [u8]> {
159    let Some(first) = input.iter().position(|&b| b == b'%') else {
160        return Cow::Borrowed(input);
161    };
162
163    let mut out = Vec::with_capacity(input.len());
164    out.extend_from_slice(&input[..first]);
165
166    let mut i = first;
167    while i < input.len() {
168        if input[i] == b'%' {
169            if i + 2 < input.len() {
170                let hi = DECODE_NIBBLE[input[i + 1] as usize];
171                let lo = DECODE_NIBBLE[input[i + 2] as usize];
172                if (hi | lo) & 0xF0 == 0 {
173                    out.push((hi << 4) | lo);
174                    i += 3;
175                    continue;
176                }
177            }
178            // Malformed or trailing `%`: pass through literally.
179            out.push(b'%');
180            i += 1;
181        } else {
182            let run_start = i;
183            while i < input.len() && input[i] != b'%' {
184                i += 1;
185            }
186            out.extend_from_slice(&input[run_start..i]);
187        }
188    }
189    Cow::Owned(out)
190}
191
192/// Errors from URL percent-decoding.
193#[derive(Debug)]
194pub enum DecodeError {
195    /// Decoded bytes are not valid UTF-8.
196    InvalidUtf8(FromUtf8Error),
197}
198
199impl Display for DecodeError {
200    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
201        match self {
202            Self::InvalidUtf8(err) => write!(f, "invalid UTF-8 in decoded bytes: {err}"),
203        }
204    }
205}
206
207impl std::error::Error for DecodeError {
208    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
209        match self {
210            Self::InvalidUtf8(err) => Some(err),
211        }
212    }
213}
214
215impl From<FromUtf8Error> for DecodeError {
216    fn from(err: FromUtf8Error) -> Self {
217        Self::InvalidUtf8(err)
218    }
219}
220
221#[cfg(test)]
222mod tests {
223    use proptest::prelude::*;
224    use rstest::rstest;
225
226    use super::*;
227
228    // RFC 3986 Section 2.3: unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
229    const UNRESERVED_CHARS: &str =
230        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~";
231
232    // RFC 3986 Section 2.2: reserved chars (gen-delims + sub-delims) must be
233    // percent-encoded when used as data.
234    const RESERVED_CHARS: &str = ":/?#[]@!$&'()*+,;=";
235
236    #[rstest]
237    #[case("", "")]
238    #[case("abc", "abc")]
239    #[case("ABC-xyz_0.9~", "ABC-xyz_0.9~")]
240    #[case(" ", "%20")]
241    #[case("+", "%2B")]
242    #[case("/", "%2F")]
243    #[case("?", "%3F")]
244    #[case("#", "%23")]
245    #[case("&", "%26")]
246    #[case("=", "%3D")]
247    #[case("%", "%25")]
248    #[case("hello world", "hello%20world")]
249    #[case("a b+c/d", "a%20b%2Bc%2Fd")]
250    // Uppercase hex per RFC 3986 Section 2.1
251    #[case("\x7f", "%7F")]
252    fn test_encode_ascii_vectors(#[case] input: &str, #[case] expected: &str) {
253        assert_eq!(encode(input), expected);
254    }
255
256    #[rstest]
257    fn test_encode_all_unreserved_unchanged() {
258        // Every char in the unreserved set should pass through.
259        let out = encode(UNRESERVED_CHARS);
260        assert_eq!(out, UNRESERVED_CHARS);
261        // And the Cow should be Borrowed (zero-copy).
262        assert!(matches!(out, Cow::Borrowed(_)));
263    }
264
265    #[rstest]
266    fn test_encode_all_reserved_percent_encoded() {
267        let out = encode(RESERVED_CHARS);
268        // Each of the 18 chars becomes a 3-byte `%HH` sequence.
269        assert_eq!(out.len(), RESERVED_CHARS.len() * 3);
270        // None of the unreserved chars, `%`, or digits A-F should appear raw
271        // in the output except as part of a `%HH` triple.
272        for byte in out.bytes() {
273            assert!(
274                matches!(byte, b'%' | b'0'..=b'9' | b'A'..=b'F'),
275                "unexpected byte {byte:#04x} in encoded reserved output"
276            );
277        }
278    }
279
280    #[rstest]
281    fn test_encode_hex_is_uppercase() {
282        // Verify RFC 3986 Section 2.1: producers SHOULD emit uppercase hex.
283        let out = encode("/");
284        assert_eq!(out, "%2F");
285        assert!(!out.contains('f'));
286    }
287
288    #[rstest]
289    fn test_encode_every_byte_position() {
290        // For each byte 0x00..=0xFF, encode a one-byte slice and verify that
291        // the output matches the spec expectation.
292        for byte in 0u8..=255 {
293            let input = [byte];
294            let out = encode_bytes(&input);
295
296            if UNRESERVED[byte as usize] {
297                assert!(
298                    matches!(out, Cow::Borrowed(_)),
299                    "unreserved byte {byte:#04x} should not allocate"
300                );
301                assert_eq!(out.as_ref(), &[byte]);
302            } else {
303                let expected = format!("%{byte:02X}").into_bytes();
304                assert_eq!(out.as_ref(), expected.as_slice(), "byte {byte:#04x}");
305            }
306        }
307    }
308
309    #[rstest]
310    fn test_encode_utf8_multibyte() {
311        // U+00E9 encoded as UTF-8 is `0xC3 0xA9` (two bytes).
312        assert_eq!(encode("\u{00E9}"), "%C3%A9");
313        // U+4E2D encoded as UTF-8 is `0xE4 0xB8 0xAD` (three bytes).
314        assert_eq!(encode("\u{4E2D}"), "%E4%B8%AD");
315        // Grinning face emoji U+1F600 is `0xF0 0x9F 0x98 0x80` (four bytes).
316        assert_eq!(encode("\u{1F600}"), "%F0%9F%98%80");
317    }
318
319    #[rstest]
320    fn test_encode_mixed_ascii_and_utf8() {
321        assert_eq!(encode("a é/"), "a%20%C3%A9%2F");
322    }
323
324    #[rstest]
325    fn test_encode_returns_borrowed_when_no_work() {
326        let out = encode("safe-string_123.xyz~");
327        assert!(matches!(out, Cow::Borrowed(_)));
328    }
329
330    #[rstest]
331    fn test_encode_returns_owned_when_encoding_needed() {
332        let out = encode("needs encoding");
333        assert!(matches!(out, Cow::Owned(_)));
334    }
335
336    #[rstest]
337    #[case("", "")]
338    #[case("abc", "abc")]
339    #[case("%20", " ")]
340    #[case("%2F", "/")]
341    #[case("%2f", "/")] // lowercase hex must be accepted
342    #[case("%2b", "+")]
343    #[case("%25", "%")]
344    #[case("hello%20world", "hello world")]
345    #[case("a%20b%2Bc%2Fd", "a b+c/d")]
346    #[case("%C3%A9", "\u{00E9}")]
347    #[case("%E4%B8%AD", "\u{4E2D}")]
348    #[case("%F0%9F%98%80", "\u{1F600}")]
349    fn test_decode_ascii_and_utf8_vectors(#[case] input: &str, #[case] expected: &str) {
350        assert_eq!(decode(input).unwrap(), expected);
351    }
352
353    #[rstest]
354    #[case("%", "%")] // bare `%` at end passes through
355    #[case("%2", "%2")] // one hex digit at end
356    #[case("%GG", "%GG")] // non-hex digits
357    #[case("%2G", "%2G")] // second nibble invalid
358    #[case("%G2", "%G2")] // first nibble invalid
359    #[case("%%20", "% ")] // first `%` literal, then `%20` decodes
360    #[case("100%", "100%")] // `%` at end after ASCII
361    fn test_decode_malformed_percent_passes_through(#[case] input: &str, #[case] expected: &str) {
362        assert_eq!(decode(input).unwrap(), expected);
363    }
364
365    #[rstest]
366    fn test_decode_returns_borrowed_when_no_percent() {
367        let out = decode("no-percent-here").unwrap();
368        assert!(matches!(out, Cow::Borrowed(_)));
369    }
370
371    #[rstest]
372    fn test_decode_returns_owned_when_percent_present() {
373        let out = decode("a%20b").unwrap();
374        assert!(matches!(out, Cow::Owned(_)));
375    }
376
377    #[rstest]
378    #[case("this%2x%26that", "this%2x&that")]
379    #[case("%%25", "%%")]
380    #[case("%2%26", "%2&")]
381    #[case("a%2Zb%20c", "a%2Zb c")]
382    fn test_decode_malformed_then_valid(#[case] input: &str, #[case] expected: &str) {
383        assert_eq!(decode(input).unwrap(), expected);
384    }
385
386    #[rstest]
387    fn test_decode_invalid_utf8_errors() {
388        // `0xFF` is not valid UTF-8 on its own.
389        let err = decode("%FF").unwrap_err();
390        assert!(matches!(err, DecodeError::InvalidUtf8(_)));
391    }
392
393    #[rstest]
394    fn test_decode_invalid_utf8_bytes_ok() {
395        // `decode_bytes` does not validate UTF-8.
396        let out = decode_bytes(b"%FF");
397        assert_eq!(out.as_ref(), &[0xFF]);
398    }
399
400    #[rstest]
401    fn test_decode_consecutive_percent_triples() {
402        // Three consecutive `%HH` sequences decoding multi-byte UTF-8.
403        assert_eq!(decode("%e2%98%83").unwrap(), "\u{2603}"); // snowman U+2603
404    }
405
406    #[rstest]
407    fn test_decode_nul_byte() {
408        // `%00` decodes to the NUL byte, which is valid UTF-8 (U+0000).
409        let decoded = decode("a%00b").unwrap();
410        assert_eq!(decoded.as_bytes(), &[b'a', 0x00, b'b']);
411    }
412
413    #[rstest]
414    fn test_roundtrip_every_byte() {
415        // For every byte 0x00..=0xFF, encoding then decoding must recover
416        // the original byte exactly.
417        for byte in 0u8..=255 {
418            let input = [byte];
419            let encoded = encode_bytes(&input);
420            let decoded = decode_bytes(encoded.as_ref());
421            assert_eq!(
422                decoded.as_ref(),
423                input.as_slice(),
424                "round-trip failed for byte {byte:#04x}"
425            );
426        }
427    }
428
429    #[rstest]
430    #[case("hello")]
431    #[case("a b c")]
432    #[case("https://example.com/path?q=1&x=2")]
433    #[case("\u{00E9}\u{00E0}\u{00FC}")]
434    #[case("\u{4E2D}\u{6587}\u{6D4B}\u{8BD5}")]
435    #[case("mix 123 !@# %^&*()")]
436    #[case("\u{1F600}\u{1F680}\u{1F3C6}")]
437    fn test_roundtrip_string(#[case] input: &str) {
438        let encoded = encode(input);
439        let decoded = decode(&encoded).unwrap();
440        assert_eq!(decoded, input);
441    }
442
443    #[rstest]
444    fn test_encoded_output_only_ascii() {
445        // Encoded output must always be pure ASCII (unreserved bytes + `%HH`).
446        let encoded = encode("\u{00E9}\u{4E2D}\u{1F600}");
447        assert!(encoded.is_ascii(), "encoded output must be ASCII-only");
448    }
449
450    #[rstest]
451    fn test_encode_bytes_arbitrary_binary() {
452        // Encoding arbitrary bytes (including non-UTF-8) yields a valid
453        // percent-encoded ASCII sequence.
454        let input: Vec<u8> = (0u8..=255).collect();
455        let encoded = encode_bytes(&input);
456        assert!(encoded.iter().all(u8::is_ascii));
457        let decoded = decode_bytes(encoded.as_ref());
458        assert_eq!(decoded.as_ref(), input.as_slice());
459    }
460
461    #[rstest]
462    fn test_decode_error_display_and_source() {
463        let err = decode("%FF").unwrap_err();
464        let msg = err.to_string();
465        assert!(msg.starts_with("invalid UTF-8"), "got: {msg}");
466        assert!(std::error::Error::source(&err).is_some());
467    }
468
469    // Independent reference implementation used to cross-check our tuned
470    // implementation on random inputs. Pure-Rust, loop-based, no table
471    // lookups: if both agree across thousands of random inputs we have
472    // strong evidence the tuned version is spec-correct.
473    fn reference_encode(input: &[u8]) -> Vec<u8> {
474        let mut out = Vec::with_capacity(input.len());
475        for &b in input {
476            let is_unreserved =
477                b.is_ascii_alphanumeric() || b == b'-' || b == b'.' || b == b'_' || b == b'~';
478            if is_unreserved {
479                out.push(b);
480            } else {
481                out.push(b'%');
482                out.extend_from_slice(format!("{b:02X}").as_bytes());
483            }
484        }
485        out
486    }
487
488    fn reference_decode(input: &[u8]) -> Vec<u8> {
489        let mut out = Vec::with_capacity(input.len());
490        let mut i = 0;
491        while i < input.len() {
492            if input[i] == b'%' && i + 2 < input.len() {
493                let a = input[i + 1];
494                let b = input[i + 2];
495                if a.is_ascii_hexdigit() && b.is_ascii_hexdigit() {
496                    let hi = if a.is_ascii_digit() {
497                        a - b'0'
498                    } else {
499                        (a | 0x20) - b'a' + 10
500                    };
501                    let lo = if b.is_ascii_digit() {
502                        b - b'0'
503                    } else {
504                        (b | 0x20) - b'a' + 10
505                    };
506                    out.push((hi << 4) | lo);
507                    i += 3;
508                    continue;
509                }
510            }
511            out.push(input[i]);
512            i += 1;
513        }
514        out
515    }
516
517    fn malformed_percent_sequence() -> impl Strategy<Value = Vec<u8>> {
518        prop_oneof![
519            Just(vec![b'%']),
520            (any::<u8>(), any::<u8>())
521                .prop_filter("contains a non-hex byte", |(hi, lo)| {
522                    !hi.is_ascii_hexdigit() || !lo.is_ascii_hexdigit()
523                })
524                .prop_map(|(hi, lo)| vec![b'%', hi, lo]),
525        ]
526    }
527
528    proptest::proptest! {
529        #[rstest]
530        fn prop_encode_matches_reference(input: Vec<u8>) {
531            let actual = encode_bytes(&input);
532            let expected = reference_encode(&input);
533            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
534        }
535
536        #[rstest]
537        fn prop_decode_matches_reference(input: Vec<u8>) {
538            let actual = decode_bytes(&input);
539            let expected = reference_decode(&input);
540            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
541        }
542
543        #[rstest]
544        fn prop_malformed_percent_sequences_match_reference(
545            prefix in proptest::collection::vec(any::<u8>(), 0..16),
546            malformed in malformed_percent_sequence(),
547            suffix in proptest::collection::vec(any::<u8>(), 0..16),
548        ) {
549            let mut input = prefix;
550            input.extend(malformed);
551            input.extend(suffix);
552
553            let actual = decode_bytes(&input);
554            let expected = reference_decode(&input);
555            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
556        }
557
558        #[rstest]
559        fn prop_bytes_roundtrip(input: Vec<u8>) {
560            let encoded = encode_bytes(&input);
561            let decoded = decode_bytes(encoded.as_ref());
562            proptest::prop_assert_eq!(decoded.as_ref(), input.as_slice());
563        }
564
565        #[rstest]
566        fn prop_string_roundtrip(input: String) {
567            let encoded = encode(&input);
568            let decoded = decode(&encoded).unwrap();
569            proptest::prop_assert_eq!(decoded.as_ref(), input.as_str());
570        }
571
572        #[rstest]
573        fn prop_encoded_output_ascii(input: Vec<u8>) {
574            let encoded = encode_bytes(&input);
575            proptest::prop_assert!(encoded.iter().all(u8::is_ascii));
576        }
577    }
578}
nautilus_core/string/urlencoding.rs

nautilus_core/string/
urlencoding.rs
