robin-sparkless 0.11.5

PySpark-like DataFrame API in Rust on Polars; no JVM.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

"""Tests for issue #217: String-to-int cast empty/invalid strings.

Casting empty or invalid strings to int should yield null (Spark/Sparkless parity),
not raise RuntimeError. Polars strict conversion is replaced by custom parsing.
"""

import robin_sparkless as rs


def test_cast_empty_and_whitespace_string_to_int() -> None:
    """Exact scenario from #217: cast('') and cast(' ') -> null."""
    spark = rs.SparkSession.builder().app_name("test").get_or_create()
    df = spark.createDataFrame(
        [{"text": ""}, {"text": " "}],
        [("text", "string")],
    )
    result = df.with_column("n", rs.col("text").cast("int"))
    rows = result.collect()
    assert len(rows) == 2
    assert rows[0]["text"] == ""
    assert rows[0]["n"] is None
    assert rows[1]["text"] == " "
    assert rows[1]["n"] is None


def test_cast_invalid_strings_to_int_null() -> None:
    """#217 affected: hello, abc123, '' -> null."""
    spark = rs.SparkSession.builder().app_name("test").get_or_create()
    df = spark.createDataFrame(
        [{"s": "hello"}, {"s": "abc123"}, {"s": ""}, {"s": "42"}],
        [("s", "string")],
    )
    result = df.with_column("n", rs.col("s").cast("int"))
    rows = result.collect()
    assert len(rows) == 4
    assert rows[0]["n"] is None
    assert rows[1]["n"] is None
    assert rows[2]["n"] is None
    assert rows[3]["n"] == 42


def test_try_cast_invalid_to_int_null() -> None:
    """try_cast also yields null for invalid."""
    spark = rs.SparkSession.builder().app_name("test").get_or_create()
    df = spark.createDataFrame(
        [{"s": "1"}, {"s": "x"}],
        [("s", "string")],
    )
    result = df.with_column("n", rs.try_cast(rs.col("s"), "int"))
    rows = result.collect()
    assert rows[0]["n"] == 1
    assert rows[1]["n"] is None


def test_cast_valid_string_to_long() -> None:
    """cast to long/bigint works; invalid -> null."""
    spark = rs.SparkSession.builder().app_name("test").get_or_create()
    df = spark.createDataFrame(
        [{"s": "9999999999"}, {"s": "bad"}],
        [("s", "string")],
    )
    result = df.with_column("n", rs.col("s").cast("long"))
    rows = result.collect()
    assert rows[0]["n"] == 9999999999
    assert rows[1]["n"] is None