robin-sparkless 4.4.0

PySpark-like DataFrame API in Rust on Polars; no JVM.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

"""
Tests for #416: createDataFrame accept pandas.DataFrame (PySpark parity).

PySpark createDataFrame accepts pandas.DataFrame; Robin-sparkless converts via to_dict("records").
"""

from __future__ import annotations

import pytest


pandas = pytest.importorskip("pandas")


def test_create_dataframe_from_pandas(spark) -> None:
    """spark.createDataFrame(pandas.DataFrame) works."""
    pdf = pandas.DataFrame({"a": [1, 2], "b": [3, 4]})
    df = spark.createDataFrame(pdf)
    out = df.collect()
    assert len(out) == 2

    # Row or dict both acceptable (sparkless returns Row, native returns dict)
    def v(r, k):
        return r[k] if isinstance(r, dict) else getattr(r, k)

    assert v(out[0], "a") == 1 and v(out[0], "b") == 3
    assert v(out[1], "a") == 2 and v(out[1], "b") == 4