datafusion_spark/function/url/
url_decode.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::borrow::Cow;
19use std::sync::Arc;
20
21use arrow::array::{
22    Array, ArrayRef, LargeStringBuilder, StringBuilder, StringViewBuilder,
23};
24use arrow::datatypes::DataType;
25use datafusion_common::cast::{
26    as_large_string_array, as_string_array, as_string_view_array,
27};
28use datafusion_common::{Result, exec_datafusion_err, exec_err, plan_err};
29use datafusion_expr::{
30    ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
31};
32use datafusion_functions::utils::make_scalar_function;
33use percent_encoding::percent_decode;
34
35#[derive(Debug, PartialEq, Eq, Hash)]
36pub struct UrlDecode {
37    signature: Signature,
38}
39
40impl Default for UrlDecode {
41    fn default() -> Self {
42        Self::new()
43    }
44}
45
46impl UrlDecode {
47    pub fn new() -> Self {
48        Self {
49            signature: Signature::string(1, Volatility::Immutable),
50        }
51    }
52
53    /// Decodes a URL-encoded string from application/x-www-form-urlencoded format.
54    /// Although the `url::form_urlencoded` support decoding, it does not return error when the string is malformed
55    ///     For example: "%2s" is not a valid percent-encoding, the `decode` function from `url::form_urlencoded`
56    ///                  will ignore this instead of return error
57    /// This function reproduce the same decoding process, plus an extra validation step
58    /// See <https://github.com/servo/rust-url/blob/b06048d70d4cc9cf4ffb277f06cfcebd53b2141e/form_urlencoded/src/lib.rs#L70-L76>
59    ///
60    /// # Arguments
61    ///
62    /// * `value` - The URL-encoded string to decode
63    ///
64    /// # Returns
65    ///
66    /// * `Ok(Cow<str>)` - The decoded string, borrowed from `value` when there
67    ///   was nothing to rewrite and owned otherwise
68    /// * `Err(DataFusionError)` - If the input is malformed or contains invalid UTF-8
69    fn decode(value: &str) -> Result<Cow<'_, str>> {
70        // Check if the string has valid percent encoding
71        Self::validate_percent_encoding(value)?;
72
73        match Self::replace_plus(value.as_bytes()) {
74            // No '+' was rewritten, so the decode can borrow from `value` itself.
75            Cow::Borrowed(bytes) => percent_decode(bytes)
76                .decode_utf8()
77                .map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")),
78            // Rewriting '+' already allocated, so owning the decoded form here
79            // costs nothing beyond what has been spent.
80            Cow::Owned(bytes) => percent_decode(&bytes)
81                .decode_utf8()
82                .map(|decoded| Cow::Owned(decoded.into_owned()))
83                .map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")),
84        }
85    }
86
87    /// Replace b'+' with b' '
88    /// See: <https://github.com/servo/rust-url/blob/dbd526178ed9276176602dd039022eba89e8fc93/form_urlencoded/src/lib.rs#L79-L93>
89    fn replace_plus(input: &[u8]) -> Cow<'_, [u8]> {
90        match input.iter().position(|&b| b == b'+') {
91            None => Cow::Borrowed(input),
92            Some(first_position) => {
93                let mut replaced = input.to_owned();
94                replaced[first_position] = b' ';
95                for byte in &mut replaced[first_position + 1..] {
96                    if *byte == b'+' {
97                        *byte = b' ';
98                    }
99                }
100                Cow::Owned(replaced)
101            }
102        }
103    }
104
105    /// Validate percent-encoding of the string
106    fn validate_percent_encoding(value: &str) -> Result<()> {
107        let bytes = value.as_bytes();
108        let mut i = 0;
109
110        while i < bytes.len() {
111            if bytes[i] == b'%' {
112                // Check if we have at least 2 more characters
113                if i + 2 >= bytes.len() {
114                    return exec_err!(
115                        "Invalid percent-encoding: incomplete sequence at position {}",
116                        i
117                    );
118                }
119
120                let hex1 = bytes[i + 1];
121                let hex2 = bytes[i + 2];
122
123                if !hex1.is_ascii_hexdigit() || !hex2.is_ascii_hexdigit() {
124                    return exec_err!(
125                        "Invalid percent-encoding: invalid hex sequence '%{}{}' at position {}",
126                        hex1 as char,
127                        hex2 as char,
128                        i
129                    );
130                }
131                i += 3;
132            } else {
133                i += 1;
134            }
135        }
136        Ok(())
137    }
138}
139
140impl ScalarUDFImpl for UrlDecode {
141    fn name(&self) -> &str {
142        "url_decode"
143    }
144
145    fn signature(&self) -> &Signature {
146        &self.signature
147    }
148
149    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
150        if arg_types.len() != 1 {
151            return plan_err!(
152                "{} expects 1 argument, but got {}",
153                self.name(),
154                arg_types.len()
155            );
156        }
157        // As the type signature is already checked, we can safely return the type of the first argument
158        Ok(arg_types[0].clone())
159    }
160
161    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
162        let ScalarFunctionArgs { args, .. } = args;
163        make_scalar_function(spark_url_decode, vec![])(&args)
164    }
165}
166
167/// How [`spark_handled_url_decode`] reacts to a malformed input value.
168#[derive(Clone, Copy, Debug, PartialEq, Eq)]
169pub enum OnDecodeError {
170    /// Propagate the error, as `url_decode` does.
171    Fail,
172    /// Return NULL for that row, as `try_url_decode` does.
173    Null,
174}
175
176/// Core implementation of URL decoding function.
177///
178/// # Arguments
179///
180/// * `args` - A slice containing exactly one ArrayRef with the URL-encoded strings to decode
181///
182/// # Returns
183///
184/// * `Ok(ArrayRef)` - A new array of the same type containing decoded strings
185/// * `Err(DataFusionError)` - If validation fails or invalid arguments are provided
186fn spark_url_decode(args: &[ArrayRef]) -> Result<ArrayRef> {
187    spark_handled_url_decode(args, OnDecodeError::Fail)
188}
189
190pub fn spark_handled_url_decode(
191    args: &[ArrayRef],
192    on_error: OnDecodeError,
193) -> Result<ArrayRef> {
194    if args.len() != 1 {
195        return exec_err!("`url_decode` expects 1 argument");
196    }
197
198    // Decoded values go straight into the builder, so a row that needs no
199    // unescaping is copied once rather than materialised as its own `String`.
200    macro_rules! decode_all {
201        ($array:expr, $builder:expr) => {{
202            let array = $array;
203            let mut builder = $builder;
204            for value in array.iter() {
205                let Some(value) = value else {
206                    builder.append_null();
207                    continue;
208                };
209                match UrlDecode::decode(value) {
210                    Ok(decoded) => builder.append_value(&decoded),
211                    Err(e) => match on_error {
212                        OnDecodeError::Fail => return Err(e),
213                        OnDecodeError::Null => builder.append_null(),
214                    },
215                }
216            }
217            Ok(Arc::new(builder.finish()) as ArrayRef)
218        }};
219    }
220
221    match &args[0].data_type() {
222        DataType::Utf8 => {
223            let array = as_string_array(&args[0])?;
224            let builder =
225                StringBuilder::with_capacity(array.len(), array.value_data().len());
226            decode_all!(array, builder)
227        }
228        DataType::LargeUtf8 => {
229            let array = as_large_string_array(&args[0])?;
230            let builder =
231                LargeStringBuilder::with_capacity(array.len(), array.value_data().len());
232            decode_all!(array, builder)
233        }
234        DataType::Utf8View => {
235            let array = as_string_view_array(&args[0])?;
236            let builder = StringViewBuilder::with_capacity(array.len());
237            decode_all!(array, builder)
238        }
239        other => exec_err!("`url_decode`: Expr must be STRING, got {other:?}"),
240    }
241}
242
243#[cfg(test)]
244mod tests {
245
246    use super::*;
247    use arrow::array::{LargeStringArray, StringArray, StringViewArray};
248
249    const INPUT: [Option<&str>; 7] = [
250        Some("https%3A%2F%2Fspark.apache.org"),
251        Some("inva+lid://user:pass@host/file\\;param?query\\;p2"),
252        Some("inva lid://user:pass@host/file\\;param?query\\;p2"),
253        Some("%7E%21%40%23%24%25%5E%26%2A%28%29%5F%2B"),
254        Some("%E4%BD%A0%E5%A5%BD"),
255        Some(""),
256        None,
257    ];
258
259    const EXPECTED: [Option<&str>; 7] = [
260        Some("https://spark.apache.org"),
261        Some("inva lid://user:pass@host/file\\;param?query\\;p2"),
262        Some("inva lid://user:pass@host/file\\;param?query\\;p2"),
263        Some("~!@#$%^&*()_+"),
264        Some("你好"),
265        Some(""),
266        None,
267    ];
268
269    // '%2s' is not a valid percent encoded character
270    const MALFORMED_INPUT: [Option<&str>; 3] = [
271        Some("http%3A%2F%2spark.apache.org"),
272        // Valid cases
273        Some("https%3A%2F%2Fspark.apache.org"),
274        None,
275    ];
276
277    #[test]
278    fn test_decode_utf8() -> Result<()> {
279        let input = Arc::new(StringArray::from(INPUT.to_vec())) as ArrayRef;
280        let result = spark_url_decode(&[input])?;
281        let result = as_string_array(&result)?;
282        assert_eq!(&StringArray::from(EXPECTED.to_vec()), result);
283        Ok(())
284    }
285
286    #[test]
287    fn test_decode_large_utf8() -> Result<()> {
288        let input = Arc::new(LargeStringArray::from(INPUT.to_vec())) as ArrayRef;
289        let result = spark_url_decode(&[input])?;
290        let result = as_large_string_array(&result)?;
291        assert_eq!(&LargeStringArray::from(EXPECTED.to_vec()), result);
292        Ok(())
293    }
294
295    #[test]
296    fn test_decode_utf8_view() -> Result<()> {
297        let input = Arc::new(StringViewArray::from(INPUT.to_vec())) as ArrayRef;
298        let result = spark_url_decode(&[input])?;
299        let result = as_string_view_array(&result)?;
300        assert_eq!(&StringViewArray::from(EXPECTED.to_vec()), result);
301        Ok(())
302    }
303
304    #[test]
305    fn test_decode_error() -> Result<()> {
306        let inputs: [ArrayRef; 3] = [
307            Arc::new(StringArray::from(MALFORMED_INPUT.to_vec())),
308            Arc::new(LargeStringArray::from(MALFORMED_INPUT.to_vec())),
309            Arc::new(StringViewArray::from(MALFORMED_INPUT.to_vec())),
310        ];
311
312        for input in inputs {
313            let result = spark_url_decode(&[input]);
314            assert!(
315                result.is_err_and(|e| e.to_string().contains("Invalid percent-encoding"))
316            );
317        }
318
319        Ok(())
320    }
321}
datafusion_spark/function/url/url_decode.rs

datafusion_spark/function/url/
url_decode.rs
