robin-sparkless 4.4.0

PySpark-like DataFrame API in Rust on Polars; no JVM.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

"""
PySpark parity tests for datetime functions.

Tests validate that Sparkless datetime functions behave identically to PySpark.
"""

from tests.tools.parity_base import ParityTestBase
from sparkless.testing import get_imports


class TestDatetimeFunctionsParity(ParityTestBase):
    """Test datetime function parity with PySpark."""

    def test_year(self, spark):
        """Test year function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "year")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.year(df.hire_date))
        self.assert_parity(result, expected)

    def test_month(self, spark):
        """Test month function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "month")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.month(df.hire_date))
        self.assert_parity(result, expected)

    def test_dayofmonth(self, spark):
        """Test dayofmonth function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "dayofmonth")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.dayofmonth(df.date))
        self.assert_parity(result, expected)

    def test_dayofweek(self, spark):
        """Test dayofweek function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "dayofweek")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.dayofweek(df.hire_date))
        self.assert_parity(result, expected)

    def test_date_add(self, spark):
        """Test date_add function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "date_add")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.date_add(df.hire_date, 30))
        self.assert_parity(result, expected)

    def test_date_sub(self, spark):
        """Test date_sub function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "date_sub")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.date_sub(df.hire_date, 30))
        self.assert_parity(result, expected)

    def test_date_format(self, spark):
        """Test date_format function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "date_format")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.date_format(df.hire_date, "yyyy-MM"))
        self.assert_parity(result, expected)

    def test_to_date(self, spark):
        """Test to_date function matches PySpark behavior."""
        imports = get_imports()
        F = imports.F
        expected = self.load_expected("datetime", "to_date")
        df = spark.createDataFrame(expected["input_data"])
        result = df.select(F.to_date(df.hire_date))
        self.assert_parity(result, expected)