spark_connect_core/
storage.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
//! Enum for handling Spark Storage representations

use crate::spark;

#[derive(Clone, Copy, Debug)]
pub enum StorageLevel {
    None,
    DiskOnly,
    DiskOnly2,
    DiskOnly3,
    MemoryOnly,
    MemoryOnly2,
    MemoryAndDisk,
    MemoryAndDisk2,
    OffHeap,
    MemoryAndDiskDeser,
}

impl From<spark::StorageLevel> for StorageLevel {
    fn from(spark_level: spark::StorageLevel) -> Self {
        match (
            spark_level.use_disk,
            spark_level.use_memory,
            spark_level.use_off_heap,
            spark_level.deserialized,
            spark_level.replication,
        ) {
            (false, false, false, false, _) => StorageLevel::None,
            (true, false, false, false, 1) => StorageLevel::DiskOnly,
            (true, false, false, false, 2) => StorageLevel::DiskOnly2,
            (true, false, false, false, 3) => StorageLevel::DiskOnly3,
            (false, true, false, false, 1) => StorageLevel::MemoryOnly,
            (false, true, false, false, 2) => StorageLevel::MemoryOnly2,
            (true, true, false, false, 1) => StorageLevel::MemoryAndDisk,
            (true, true, false, false, 2) => StorageLevel::MemoryAndDisk2,
            (true, true, true, false, 1) => StorageLevel::OffHeap,
            (true, true, false, true, 1) => StorageLevel::MemoryAndDiskDeser,
            _ => unimplemented!(),
        }
    }
}

impl From<StorageLevel> for spark::StorageLevel {
    fn from(storage: StorageLevel) -> spark::StorageLevel {
        match storage {
            StorageLevel::None => spark::StorageLevel {
                use_disk: false,
                use_memory: false,
                use_off_heap: false,
                deserialized: false,
                replication: 1,
            },
            StorageLevel::DiskOnly => spark::StorageLevel {
                use_disk: true,
                use_memory: false,
                use_off_heap: false,
                deserialized: false,
                replication: 1,
            },
            StorageLevel::DiskOnly2 => spark::StorageLevel {
                use_disk: true,
                use_memory: false,
                use_off_heap: false,
                deserialized: false,
                replication: 2,
            },
            StorageLevel::DiskOnly3 => spark::StorageLevel {
                use_disk: true,
                use_memory: false,
                use_off_heap: false,
                deserialized: false,
                replication: 3,
            },
            StorageLevel::MemoryOnly => spark::StorageLevel {
                use_disk: false,
                use_memory: true,
                use_off_heap: false,
                deserialized: false,
                replication: 1,
            },
            StorageLevel::MemoryOnly2 => spark::StorageLevel {
                use_disk: false,
                use_memory: true,
                use_off_heap: false,
                deserialized: false,
                replication: 2,
            },
            StorageLevel::MemoryAndDisk => spark::StorageLevel {
                use_disk: true,
                use_memory: true,
                use_off_heap: false,
                deserialized: false,
                replication: 1,
            },
            StorageLevel::MemoryAndDisk2 => spark::StorageLevel {
                use_disk: true,
                use_memory: true,
                use_off_heap: false,
                deserialized: false,
                replication: 2,
            },
            StorageLevel::OffHeap => spark::StorageLevel {
                use_disk: true,
                use_memory: true,
                use_off_heap: true,
                deserialized: false,
                replication: 1,
            },
            StorageLevel::MemoryAndDiskDeser => spark::StorageLevel {
                use_disk: true,
                use_memory: true,
                use_off_heap: false,
                deserialized: true,
                replication: 1,
            },
        }
    }
}