Skip to main content

spark_connect_macros/
lib.rs

1//! Procedural macros for the Rust Spark Connect client.
2//!
3//! Provides [`macro@spark_wasm_udf`], which turns a module of plain Rust
4//! functions into Spark UDFs that run on the executors via WebAssembly.
5
6use proc_macro::TokenStream;
7use quote::{format_ident, quote};
8use syn::{
9    parse_macro_input, FnArg, GenericArgument, Item, ItemFn, ItemMod, Pat, PathArguments,
10    ReturnType, Type,
11};
12
13/// Turn a module of plain Rust functions into Spark UDFs backed by WebAssembly.
14///
15/// Applied to a `mod`, for each free function inside it the macro:
16///
17/// * keeps the function callable natively,
18/// * emits (when compiled for `wasm32`) a `#[no_mangle] extern "C"` export that
19///   decodes the arguments and encodes the result with the length-prefixed
20///   binary ABI (`spark_connect::wasm_udf::AbiType`), and
21/// * generates, on non-`wasm32` targets, a direct-call constructor
22///   `udf::<name>(col0, col1, ...) -> Result<Column>` (one column per argument,
23///   arity checked at compile time) plus a builder `udf::<name>_udf() ->
24///   UserDefinedFunction` for advanced configuration — both with the Spark
25///   signature **inferred** from the Rust signature and the crate's own compiled
26///   wasm module embedded.
27///
28/// A function whose arguments and result are all `i32`, `i64`, `f32`, `f64`,
29/// `bool`, `String`, `Vec<u8>`, or one `Option` of those also gets a batch
30/// export, and its builder sends it as a scalar Arrow UDF that Spark calls once
31/// per Arrow batch. `.with_eval_type(spark_connect::udf::eval_type::SQL_ARROW_BATCHED_UDF)`
32/// on the builder calls it once per row instead, like functions with other types.
33///
34/// ```ignore
35/// use spark_connect_macros::spark_wasm_udf;
36///
37/// #[spark_wasm_udf]
38/// mod udfs {
39///     pub fn add_one(x: i64) -> i64 { x + 1 }
40///     pub fn shout(s: String) -> String { format!("{}!", s.to_uppercase()) }
41/// }
42///
43/// // callers (build.rs embeds the module):
44/// df.select(vec![udf::add_one(col("id"))?]);              // direct call
45/// df.select(vec![udf::add_one_udf().as_nondeterministic().call(vec![col("id")])?]);
46/// ```
47///
48/// Supported types: `i32`, `i64`, `f32`, `f64`, `bool`, `String`, `Vec<u8>`
49/// (binary), `Vec<T>` (array), and `Option<T>` (nullable), nested arbitrarily.
50#[proc_macro_attribute]
51pub fn spark_wasm_udf(_attr: TokenStream, item: TokenStream) -> TokenStream {
52    let module = parse_macro_input!(item as ItemMod);
53    match expand(module) {
54        Ok(ts) => ts.into(),
55        Err(e) => e.to_compile_error().into(),
56    }
57}
58
59/// Map a Rust type to a `spark_connect::wasm_udf::AbiType` constructor expr.
60fn abi_type(ty: &Type) -> Result<proc_macro2::TokenStream, syn::Error> {
61    let root = quote!(::spark_connect::wasm_udf::AbiType);
62    if let Some(ident) = path_ident(ty) {
63        match ident.as_str() {
64            "i32" => return Ok(quote!(#root::I32)),
65            "i64" => return Ok(quote!(#root::I64)),
66            "f32" => return Ok(quote!(#root::F32)),
67            "f64" => return Ok(quote!(#root::F64)),
68            "bool" => return Ok(quote!(#root::Bool)),
69            "String" => return Ok(quote!(#root::Str)),
70            _ => {}
71        }
72    }
73    if let Some(inner) = generic_inner(ty, "Vec") {
74        if path_ident(inner).as_deref() == Some("u8") {
75            return Ok(quote!(#root::Binary));
76        }
77        let inner = abi_type(inner)?;
78        return Ok(quote!(#root::Array(::std::boxed::Box::new(#inner))));
79    }
80    if let Some(inner) = generic_inner(ty, "Option") {
81        let inner = abi_type(inner)?;
82        return Ok(quote!(#root::Nullable(::std::boxed::Box::new(#inner))));
83    }
84    Err(syn::Error::new_spanned(
85        ty,
86        "#[spark_wasm_udf] supports i32/i64/f32/f64/bool/String/Vec<u8>/Vec<T>/Option<T> \
87         (use owned types, e.g. `String` not `&str`)",
88    ))
89}
90
91fn batch_supported(ty: &Type) -> bool {
92    if let Some(inner) = generic_inner(ty, "Option") {
93        return generic_inner(inner, "Option").is_none() && batch_supported(inner);
94    }
95    if let Some(inner) = generic_inner(ty, "Vec") {
96        return path_ident(inner).as_deref() == Some("u8");
97    }
98    matches!(
99        path_ident(ty).as_deref(),
100        Some("i32" | "i64" | "f32" | "f64" | "bool" | "String")
101    )
102}
103
104/// The last path segment ident of a plain type path (`i64`, `String`, ...).
105fn path_ident(ty: &Type) -> Option<String> {
106    match ty {
107        Type::Path(p) if p.qself.is_none() => p.path.segments.last().map(|s| s.ident.to_string()),
108        _ => None,
109    }
110}
111
112/// If `ty` is `Wrapper<Inner>` (e.g. `Vec<T>`, `Option<T>`), return `Inner`.
113fn generic_inner<'a>(ty: &'a Type, wrapper: &str) -> Option<&'a Type> {
114    let Type::Path(p) = ty else { return None };
115    let seg = p.path.segments.last()?;
116    if seg.ident != wrapper {
117        return None;
118    }
119    let PathArguments::AngleBracketed(args) = &seg.arguments else {
120        return None;
121    };
122    args.args.iter().find_map(|a| match a {
123        GenericArgument::Type(t) => Some(t),
124        _ => None,
125    })
126}
127
128struct Udf {
129    name: syn::Ident,
130    export: syn::Ident,
131    batch_export: Option<syn::Ident>,
132    arg_names: Vec<String>,
133    arg_tys: Vec<Type>,
134    ret_ty: Type,
135}
136
137fn collect_udf(func: &ItemFn) -> Result<Udf, syn::Error> {
138    if func.sig.asyncness.is_some() {
139        return Err(syn::Error::new_spanned(
140            &func.sig,
141            "#[spark_wasm_udf] cannot be applied to async functions",
142        ));
143    }
144    if !func.sig.generics.params.is_empty() {
145        return Err(syn::Error::new_spanned(
146            &func.sig.generics,
147            "#[spark_wasm_udf] cannot be applied to generic functions",
148        ));
149    }
150    let name = func.sig.ident.clone();
151    let mut arg_names = Vec::new();
152    let mut arg_tys = Vec::new();
153    for (i, input) in func.sig.inputs.iter().enumerate() {
154        match input {
155            FnArg::Typed(pt) => {
156                arg_names.push(match &*pt.pat {
157                    Pat::Ident(p) => p.ident.to_string(),
158                    _ => format!("#{i}"),
159                });
160                arg_tys.push((*pt.ty).clone());
161            }
162            FnArg::Receiver(r) => {
163                return Err(syn::Error::new_spanned(
164                    r,
165                    "#[spark_wasm_udf] cannot be applied to methods (`self`)",
166                ))
167            }
168        }
169    }
170    if arg_tys.is_empty() {
171        return Err(syn::Error::new_spanned(
172            &func.sig,
173            "#[spark_wasm_udf] requires at least one argument",
174        ));
175    }
176    let ret_ty = match &func.sig.output {
177        ReturnType::Type(_, ty) => (**ty).clone(),
178        ReturnType::Default => {
179            return Err(syn::Error::new_spanned(
180                &func.sig,
181                "#[spark_wasm_udf] requires a return type",
182            ))
183        }
184    };
185    let export = format_ident!("spark_udf_{}", name);
186    let batch_export = arg_tys
187        .iter()
188        .chain([&ret_ty])
189        .all(batch_supported)
190        .then(|| format_ident!("spark_batch_udf_{}", name));
191    Ok(Udf {
192        name,
193        export,
194        batch_export,
195        arg_names,
196        arg_tys,
197        ret_ty,
198    })
199}
200
201fn expand(mut module: ItemMod) -> Result<proc_macro2::TokenStream, syn::Error> {
202    let mod_ident = module.ident.clone();
203    let Some((_, items)) = &module.content else {
204        return Err(syn::Error::new_spanned(
205            &module,
206            "#[spark_wasm_udf] must be applied to an inline module with a body",
207        ));
208    };
209
210    let udfs: Vec<Udf> = items
211        .iter()
212        .filter_map(|it| match it {
213            Item::Fn(f) => Some(collect_udf(f)),
214            _ => None,
215        })
216        .collect::<Result<_, _>>()?;
217    if udfs.is_empty() {
218        return Err(syn::Error::new_spanned(
219            &module,
220            "#[spark_wasm_udf] module contains no functions",
221        ));
222    }
223
224    // WASM export wrappers, injected into the module (compiled only for wasm32).
225    let mut wrappers = Vec::new();
226    for u in &udfs {
227        let Udf {
228            name,
229            export,
230            batch_export,
231            arg_names,
232            arg_tys,
233            ret_ty,
234        } = u;
235        let params: Vec<_> = arg_tys
236            .iter()
237            .map(|ty| quote!(<#ty as crate::spark_wasm_rt::Abi>::decode(&mut __r)))
238            .collect();
239        wrappers.push(quote! {
240            #[cfg(target_arch = "wasm32")]
241            #[no_mangle]
242            pub extern "C" fn #export(__ptr: *const u8, __len: u32) -> u64 {
243                let __b = unsafe { ::core::slice::from_raw_parts(__ptr, __len as usize) };
244                let mut __r = crate::spark_wasm_rt::Reader::new(__b);
245                let __out: #ret_ty = #name(#(#params),*);
246                let mut __o = ::std::vec::Vec::new();
247                crate::spark_wasm_rt::Abi::encode(&__out, &mut __o);
248                crate::spark_wasm_rt::finish(__o)
249            }
250        });
251        if let Some(batch_export) = batch_export {
252            let vars: Vec<syn::Ident> = (0..arg_tys.len())
253                .map(|i| format_ident!("__a{i}"))
254                .collect();
255            let gets = arg_tys.iter().zip(&vars).enumerate().map(|(i, (ty, v))| {
256                quote! {
257                    let #v: #ty =
258                        <#ty as crate::spark_wasm_rt::batch::FromCol>::get(&__cols[#i], __i)
259                            .ok_or(#i)?;
260                }
261            });
262            wrappers.push(quote! {
263                #[cfg(target_arch = "wasm32")]
264                #[no_mangle]
265                pub extern "C" fn #batch_export(__desc: u32) -> u32 {
266                    crate::spark_wasm_rt::batch::run(__desc, &[#(#arg_names),*], |__cols, __i, __out| {
267                        #(#gets)*
268                        let __r: #ret_ty = #name(#(#vars),*);
269                        crate::spark_wasm_rt::batch::ToCol::push(&__r, __out);
270                        ::core::result::Result::Ok(())
271                    })
272                }
273            });
274        }
275    }
276    if let Some((_, content)) = &mut module.content {
277        for w in wrappers {
278            content.push(syn::parse2(w)?);
279        }
280    }
281
282    // Host-side constructors, grouped under `udf::`. For each UDF the macro
283    // emits two functions:
284    //   * `udf::<name>(col0, col1, ...) -> Result<Column>` — the direct call,
285    //     one column argument per Rust parameter (arity checked at compile
286    //     time), mirroring calling a PySpark UDF on columns.
287    //   * `udf::<name>_udf() -> UserDefinedFunction` — the builder, for advanced
288    //     configuration (`.as_nondeterministic()`, `.with_packer()`, ...) before
289    //     `.call(cols)`.
290    let mut ctors = Vec::new();
291    for u in &udfs {
292        let Udf {
293            name,
294            export,
295            batch_export,
296            arg_tys,
297            ret_ty,
298            ..
299        } = u;
300        let name_str = name.to_string();
301        let export_str = export.to_string();
302        let builder = format_ident!("{}_udf", name);
303        let arg_abis: Vec<_> = arg_tys.iter().map(abi_type).collect::<Result<_, _>>()?;
304        let ret_abi = abi_type(ret_ty)?;
305        let with_batch = batch_export.as_ref().map(|b| {
306            let b = b.to_string();
307            quote!(.with_batch_entrypoint(#b))
308        });
309        let params: Vec<syn::Ident> = (0..arg_tys.len())
310            .map(|i| format_ident!("__a{i}"))
311            .collect();
312        let builder_doc = format!(
313            "Builder for the `{name_str}` UDF, generated by `#[spark_wasm_udf]`. \
314             For advanced configuration (`.as_nondeterministic()`, `.with_packer()`) \
315             before `.call(cols)`; the common case is `{name_str}(cols...)`."
316        );
317        let call_doc = format!(
318            "Apply the `{name_str}` UDF to columns, generated by `#[spark_wasm_udf]`. \
319             Takes one column per argument and returns the result column."
320        );
321        ctors.push(quote! {
322            #[doc = #builder_doc]
323            pub fn #builder() -> ::spark_connect::wasm_udf::UserDefinedFunction {
324                const __MODULE: &[u8] = include_bytes!(env!("WASM_UDFS_MODULE"));
325                ::spark_connect::wasm_udf::udf(
326                    #name_str,
327                    __MODULE.to_vec(),
328                    #export_str,
329                    ::std::vec![#(#arg_abis),*],
330                    #ret_abi,
331                )
332                #with_batch
333            }
334
335            #[doc = #call_doc]
336            pub fn #name(
337                #(#params: ::spark_connect::Column),*
338            ) -> ::spark_connect::Result<::spark_connect::Column> {
339                #builder().call(::std::vec![#(#params),*])
340            }
341        });
342    }
343
344    let doc =
345        format!("UDF constructors generated by `#[spark_wasm_udf]` from module `{mod_ident}`.");
346    Ok(quote! {
347        // Off wasm32 the exported UDF functions are called only from the
348        // `#[cfg(target_arch = "wasm32")]` wrappers injected above, so on the host
349        // build they legitimately look unused - they exist to be compiled to wasm.
350        // Allow dead_code there (not on wasm32, where the wrappers do use them) so the
351        // macro does not force `-D warnings` failures on every generated UDF module.
352        #[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))]
353        #module
354
355        #[cfg(not(target_arch = "wasm32"))]
356        #[doc = #doc]
357        pub mod udf {
358            #(#ctors)*
359        }
360    })
361}
362
363#[cfg(test)]
364mod tests {
365    use super::batch_supported;
366
367    #[test]
368    fn batch_supported_accepts_scalars_and_one_option_level_only() {
369        for (ty, expected) in [
370            ("i32", true),
371            ("i64", true),
372            ("f32", true),
373            ("f64", true),
374            ("bool", true),
375            ("String", true),
376            ("std::string::String", true),
377            ("Vec<u8>", true),
378            ("Option<i64>", true),
379            ("Option<String>", true),
380            ("Option<Vec<u8>>", true),
381            ("Vec<i64>", false),
382            ("Vec<String>", false),
383            ("Vec<Option<u8>>", false),
384            ("Option<Vec<i64>>", false),
385            ("Option<Option<i64>>", false),
386        ] {
387            let parsed: syn::Type = syn::parse_str(ty).unwrap();
388            assert_eq!(batch_supported(&parsed), expected, "{ty}");
389        }
390    }
391}