robin-sparkless 0.11.2

PySpark-like DataFrame API in Rust on Polars; no JVM.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

"""Tests for #373: SparkSession.builder().option() and .config() (PySpark parity)."""

from __future__ import annotations

import robin_sparkless as rs


def test_builder_option() -> None:
    """builder().option(key, value) stores config; spark.conf().get(key) returns value."""
    spark = (
        rs.SparkSession.builder()
        .app_name("issue_373")
        .option("spark.sql.shuffle.partitions", "2")
        .get_or_create()
    )
    assert spark.conf().get("spark.sql.shuffle.partitions") == "2"


def test_builder_config() -> None:
    """builder().config(key, value) stores config; spark.conf().get(key) returns value."""
    spark = (
        rs.SparkSession.builder()
        .app_name("issue_373_config")
        .config("custom.key", "custom.value")
        .get_or_create()
    )
    assert spark.conf().get("custom.key") == "custom.value"


def test_builder_option_and_config_chain() -> None:
    """builder() can chain .option() and .config() before get_or_create()."""
    spark = (
        rs.SparkSession.builder()
        .app_name("issue_373_chain")
        .option("a", "1")
        .config("b", "2")
        .get_or_create()
    )
    assert spark.conf().get("a") == "1"
    assert spark.conf().get("b") == "2"