apache-spark-connect 4.2.0

Pure-Rust Spark Connect DataFrame client mirroring the PySpark API surface
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! Exercise the newer Spark 4.2 SQL function wrappers. Each builds a `Column` by
//! constructing an expression (no server needed), so calling them pins their bodies.

use spark_connect::functions as f;
use spark_connect::functions::col;

#[test]
fn newer_function_builders() {
    let c = || col("a");
    let d = || col("b");

    // time / timestamp constructors and converters
    let _ = f::make_time(c(), c(), c());
    let _ = f::make_timestamp(c(), c(), c(), c(), c(), c());
    let _ = f::make_timestamp_ltz(c(), c(), c(), c(), c(), c());
    let _ = f::make_timestamp_ntz(c(), c(), c(), c(), c(), c());
    let _ = f::try_make_timestamp(c(), c(), c(), c(), c(), c());
    let _ = f::try_make_timestamp_ltz(c(), c(), c(), c(), c(), c());
    let _ = f::try_make_timestamp_ntz(c(), c(), c(), c(), c(), c());
    let _ = f::to_time(c());
    let _ = f::try_to_time(c());
    let _ = f::try_to_date(c());
    let _ = f::timestamp_nanos(c());
    let _ = f::unix_nanos(c());
    let _ = f::time_bucket(c(), d());
    let _ = f::time_diff(c(), c(), d());
    let _ = f::time_from_micros(c());
    let _ = f::time_from_millis(c());
    let _ = f::time_from_seconds(c());
    let _ = f::time_to_micros(c());
    let _ = f::time_to_millis(c());
    let _ = f::time_to_seconds(c());
    let _ = f::time_trunc(c(), d());
    let _ = f::current_time();
    let _ = f::current_path();

    // geospatial
    let _ = f::st_asbinary(c());
    let _ = f::st_geogfromwkb(c());
    let _ = f::st_geomfromwkb(c());
    let _ = f::st_setsrid(c(), d());
    let _ = f::st_srid(c());

    // theta sketches
    let _ = f::theta_difference(c(), d());
    let _ = f::theta_intersection(c(), d());
    let _ = f::theta_intersection_agg(c());
    let _ = f::theta_sketch_agg(c());
    let _ = f::theta_sketch_estimate(c());
    let _ = f::theta_union(c(), d());
    let _ = f::theta_union_agg(c());

    // KLL sketches (bigint/double/float families)
    let _ = f::kll_merge_agg_bigint(c());
    let _ = f::kll_merge_agg_double(c());
    let _ = f::kll_merge_agg_float(c());
    let _ = f::kll_sketch_agg_bigint(c());
    let _ = f::kll_sketch_agg_double(c());
    let _ = f::kll_sketch_agg_float(c());
    let _ = f::kll_sketch_get_n_bigint(c());
    let _ = f::kll_sketch_get_n_double(c());
    let _ = f::kll_sketch_get_n_float(c());
    let _ = f::kll_sketch_get_quantile_bigint(c(), d());
    let _ = f::kll_sketch_get_quantile_double(c(), d());
    let _ = f::kll_sketch_get_quantile_float(c(), d());
    let _ = f::kll_sketch_get_rank_bigint(c(), d());
    let _ = f::kll_sketch_get_rank_double(c(), d());
    let _ = f::kll_sketch_get_rank_float(c(), d());
    let _ = f::kll_sketch_merge_bigint(c(), d());
    let _ = f::kll_sketch_merge_double(c(), d());
    let _ = f::kll_sketch_merge_float(c(), d());
    let _ = f::kll_sketch_to_string_bigint(c());
    let _ = f::kll_sketch_to_string_double(c());
    let _ = f::kll_sketch_to_string_float(c());

    // variant
    let _ = f::is_valid_variant(c());
    let _ = f::variant_array_append(c(), c(), c());
    let _ = f::variant_delete(c(), vec![c()]);
    let _ = f::variant_insert(c(), c(), c());
    let _ = f::variant_set(c(), c(), c());
    let _ = f::try_variant_array_append(c(), c(), c());
    let _ = f::try_variant_insert(c(), c(), c());
    let _ = f::try_variant_set(c(), c(), c());

    // vector
    let _ = f::vector_avg(c());
    let _ = f::vector_cosine_similarity(c(), d());
    let _ = f::vector_inner_product(c(), d());
    let _ = f::vector_l2_distance(c(), d());

    // misc scalar/agg
    let _ = f::bitmap_and_agg(c());
    let _ = f::chr(c());
    let _ = f::counter_diff(c());
    let _ = f::hmac(c(), d());
    let _ = f::jaro_winkler_similarity(c(), d());
    let _ = f::quote(c());
    let _ = f::sha2(c(), 256);
    let _ = f::uuid();

    // avro / protobuf
    let _ = f::from_avro(c(), "{}");
    let _ = f::from_avro_with_options(c(), "{}", d());
    let _ = f::to_avro(c());
    let _ = f::to_avro_with_schema(c(), "{}");
    let _ = f::from_protobuf(c(), "M");
    let _ = f::from_protobuf_with_options(c(), "M", d());
    let _ = f::to_protobuf(c(), "M");
    let _ = f::to_protobuf_with_options(c(), "M", d());

    // higher-order (closures take the lambda's bound variables as a Vec<Column>)
    let _ = f::map_zip_with(c(), d(), |args| args[0].clone());
    let _ = f::reduce(c(), col("init"), |args| args[0].clone());
}