robin-sparkless 0.11.4

PySpark-like DataFrame API in Rust on Polars; no JVM.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

"""
Tests for #361: dropDuplicates(subset=[...]) (PySpark parity).

PySpark: df.dropDuplicates(subset=None) / df.drop_duplicates(subset=[...]). Robin-sparkless now supports both.
"""

from __future__ import annotations

import robin_sparkless as rs


def _spark() -> rs.SparkSession:
    return rs.SparkSession.builder().app_name("issue_361").get_or_create()


def test_drop_duplicates_subset_issue_repro() -> None:
    """df.drop_duplicates(subset=["a"]).collect() (issue repro)."""
    spark = _spark()
    create_df = getattr(spark, "create_dataframe_from_rows", spark.createDataFrame)
    df = create_df(
        [{"a": 1, "b": 1}, {"a": 1, "b": 2}, {"a": 2, "b": 1}],
        [("a", "int"), ("b", "int")],
    )
    rows = df.drop_duplicates(subset=["a"]).collect()
    assert len(rows) == 2
    a_vals = {r["a"] for r in rows}
    assert a_vals == {1, 2}


def test_drop_duplicates_no_subset() -> None:
    """drop_duplicates() with no subset = distinct on all columns."""
    spark = _spark()
    create_df = getattr(spark, "create_dataframe_from_rows", spark.createDataFrame)
    df = create_df(
        [{"a": 1, "b": 1}, {"a": 1, "b": 1}, {"a": 2, "b": 1}],
        [("a", "int"), ("b", "int")],
    )
    rows = df.drop_duplicates().collect()
    assert len(rows) == 2


def test_drop_duplicates_subset_multiple_columns() -> None:
    """drop_duplicates(subset=["a", "b"]) keeps one row per (a, b)."""
    spark = _spark()
    create_df = getattr(spark, "create_dataframe_from_rows", spark.createDataFrame)
    df = create_df(
        [{"a": 1, "b": 1}, {"a": 1, "b": 2}, {"a": 1, "b": 1}],
        [("a", "int"), ("b", "int")],
    )
    rows = df.drop_duplicates(subset=["a", "b"]).collect()
    assert len(rows) == 2


def test_drop_duplicates_camel_case() -> None:
    """dropDuplicates(subset=[...]) works (PySpark camelCase)."""
    spark = _spark()
    create_df = getattr(spark, "create_dataframe_from_rows", spark.createDataFrame)
    df = create_df(
        [{"a": 1, "b": 1}, {"a": 1, "b": 2}, {"a": 2, "b": 1}],
        [("a", "int"), ("b", "int")],
    )
    rows = df.dropDuplicates(subset=["a"]).collect()
    assert len(rows) == 2
    assert {r["a"] for r in rows} == {1, 2}