arrow2 0.18.0

Unofficial implementation of Apache Arrow spec in safe Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

import os

import pyorc


data = {
    "float_nullable": [1.0, 2.0, None, 4.0, 5.0],
    "float_required": [1.0, 2.0, 3.0, 4.0, 5.0],
    "bool_nullable": [True, False, None, True, False],
    "bool_required": [True, False, True, True, False],
    "int_nullable": [5, -5, None, 5, 5],
    "int_required": [5, -5, 1, 5, 5],
    "double_nullable": [1.0, 2.0, None, 4.0, 5.0],
    "double_required": [1.0, 2.0, 3.0, 4.0, 5.0],
    "bigint_nullable": [5, -5, None, 5, 5],
    "bigint_required": [5, -5, 1, 5, 5],
    "utf8_required": ["a", "bb", "ccc", "dddd", "eeeee"],
    "utf8_nullable": ["a", "bb", None, "dddd", "eeeee"],
}

def infer_schema(data):
    schema = "struct<"
    for key, value in data.items():
        dt = type(value[0])
        if dt == float:
            dt = "float"
        elif dt == int:
            dt = "int"
        elif dt == bool:
            dt = "boolean"
        elif dt == str:
            dt = "string"
        else:
            raise NotImplementedError
        if key.startswith("double"):
            dt = "double"
        if key.startswith("bigint"):
            dt = "bigint"
        schema += key + ":" + dt + ","

    schema = schema[:-1] + ">"
    return schema


def _write(
    data,
    file_name: str,
    compression=pyorc.CompressionKind.NONE,
    dict_key_size_threshold=0.0,
):
    schema = infer_schema(data)

    output = open(file_name, "wb")
    writer = pyorc.Writer(
        output,
        schema,
        dict_key_size_threshold=dict_key_size_threshold,
        compression=compression,
    )
    num_rows = len(list(data.values())[0])
    for x in range(num_rows):
        row = tuple(values[x] for values in data.values())
        writer.write(row)
    writer.close()

os.makedirs("fixtures/pyorc", exist_ok=True)
_write(data, "fixtures/pyorc/test.orc")