robin-sparkless 4.0.0

PySpark-like DataFrame API in Rust on Polars; no JVM.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

import pytest


class TestFunctionAPIs:
    """Test that function APIs match PySpark exactly."""

    def test_current_date_is_function_not_method(self, spark):
        """Test that current_date is a function, not DataFrame method."""
        from tests.fixtures.spark_imports import get_spark_imports

        imports = get_spark_imports()
        F = imports.F

        df = spark.createDataFrame([{"id": 1}], ["id"])

        # Should work as function
        result = df.withColumn("today", F.current_date())
        assert result is not None

        # Should NOT work as DataFrame method
        with pytest.raises(AttributeError):
            df.current_date()

    def test_current_timestamp_is_function_not_method(self, spark):
        """Test that current_timestamp is a function, not DataFrame method."""
        from tests.fixtures.spark_imports import get_spark_imports

        imports = get_spark_imports()
        F = imports.F

        df = spark.createDataFrame([{"id": 1}], ["id"])

        # Should work as function
        result = df.withColumn("now", F.current_timestamp())
        assert result is not None

        # Should NOT work as DataFrame method
        with pytest.raises(AttributeError):
            df.current_timestamp()

    def test_functions_are_static_methods(self, spark):
        """Test that functions are accessible as static methods."""
        from tests.fixtures.spark_imports import get_spark_imports

        imports = get_spark_imports()
        F = imports.F

        # All these should work as static method calls
        col_expr = F.col("id")
        assert col_expr is not None

        lit_expr = F.lit(42)
        assert lit_expr is not None

        count_func = F.count("id")
        assert count_func is not None

        row_num = F.row_number()
        assert row_num is not None

    def test_function_signatures_match_pyspark(self, spark):
        """Test that function signatures match PySpark patterns."""
        from tests.fixtures.spark_imports import get_spark_imports

        imports = get_spark_imports()
        F = imports.F

        df = spark.createDataFrame([{"id": 1, "value": 10}], ["id", "value"])

        # Test various function calling patterns that should match PySpark
        # Use an aggregate context for count/sum to match PySpark's requirement
        result = df.groupBy().agg(
            F.count("id").alias("count"),
            F.sum("value").alias("sum"),
            F.current_date().alias("today"),
            F.current_timestamp().alias("now"),
        )
        rows = result.collect()
        assert len(rows) == 1