icechunk-python 0.1.0-alpha.1

Transactional storage engine for Zarr designed for use on cloud object storage
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

import os

import icechunk
import pytest
import zarr


@pytest.fixture(scope="function")
async def tmp_store(tmpdir):
    store_path = f"{tmpdir}"
    store = await icechunk.IcechunkStore.open(
        storage=icechunk.StorageConfig.filesystem(store_path),
        mode="a",
        config=icechunk.StoreConfig(inline_chunk_threshold_bytes=5),
    )

    yield store, store_path

    store.close()


async def test_no_inline_chunks(tmp_store):
    store = tmp_store[0]
    store_path = tmp_store[1]
    array = zarr.open_array(
        store=store,
        mode="a",
        shape=(10),
        dtype="int64",
        zarr_format=3,
        chunk_shape=(1),
        fill_value=-1,
    )
    array[:] = 42

    # Check that the chunks directory was created, since each chunk is 4 bytes and the
    # inline_chunk_threshold is 1, we should have 10 chunks in the chunks directory
    assert os.path.isdir(f"{store_path}/chunks")
    assert len(os.listdir(f"{store_path}/chunks")) == 10


async def test_inline_chunks(tmp_store):
    store = tmp_store[0]
    store_path = tmp_store[1]

    inline_array = zarr.open_array(
        store=store,
        mode="a",
        path="inline",
        shape=(10),
        dtype="int32",
        zarr_format=3,
        chunk_shape=(1),
        fill_value=-1,
    )

    inline_array[:] = 9

    # Check that the chunks directory was not created, since each chunk is 4 bytes and the
    # inline_chunk_threshold is 40, we should have no chunks directory
    assert not os.path.isdir(f"{store_path}/chunks")

    written_array = zarr.open_array(
        store=store,
        mode="a",
        path="not_inline",
        shape=(10),
        dtype="int64",
        zarr_format=3,
        chunk_shape=(1),
        fill_value=-1,
    )

    written_array[:] = 3

    # Check that the chunks directory was not created, since each chunk is 8 bytes and the
    # inline_chunk_threshold is 40, we should have 10 chunks in the chunks directory
    assert os.path.isdir(f"{store_path}/chunks")
    assert len(os.listdir(f"/{store_path}/chunks")) == 10