supertable-core 0.1.1

Core library for SuperTable, a next-generation open table format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use crate::manifest::{DataFile, FileContent, FileFormat};
use crate::storage::Storage;
use arrow::array::{Int64Array, RecordBatch, StringArray};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use bytes::Bytes;
use parquet::arrow::ArrowWriter;
use parquet::file::properties::WriterProperties;
use std::io::Cursor;
use std::sync::Arc;

/// Writes position deletes (file_path, pos) to Parquet files.
pub struct PositionDeleteWriter {
    storage: Storage,
    table_location: String,
}

impl PositionDeleteWriter {
    pub fn new(storage: Storage, table_location: String) -> Self {
        Self {
            storage,
            table_location,
        }
    }

    /// Writes a batch of position deletes for a specific data file.
    pub async fn write_deletes(
        &self,
        target_file_path: &str,
        positions: Vec<i64>,
        file_id: &str,
    ) -> anyhow::Result<DataFile> {
        let schema = Arc::new(Schema::new(vec![
            Field::new("file_path", DataType::Utf8, false),
            Field::new("pos", DataType::Int64, false),
        ]));

        let len = positions.len();
        let file_path_array = StringArray::from(vec![target_file_path; len]);
        let pos_array = Int64Array::from(positions);

        let batch = RecordBatch::try_new(
            schema.clone(),
            vec![Arc::new(file_path_array), Arc::new(pos_array)],
        )?;

        // Write Parquet
        let props = WriterProperties::builder().build();
        let mut buffer = Cursor::new(Vec::new());
        {
            let mut writer = ArrowWriter::try_new(&mut buffer, schema.clone(), Some(props))?;
            writer.write(&batch)?;
            writer.close()?;
        }

        let data = buffer.into_inner();
        let file_size = data.len() as i64;
        let record_count = len as i64;
        let path = format!("{}/data/{}-delete.parquet", self.table_location, file_id);

        self.storage.write(&path, Bytes::from(data)).await?;

        // Format is currently Data, should be PositionDeletes
        let data_file = DataFile::new(path, FileFormat::Parquet, record_count, file_size)
            .with_content(FileContent::PositionDeletes);

        Ok(data_file)
    }
}

/// Writes equality deletes to Parquet files.
pub struct EqualityDeleteWriter {
    storage: Storage,
    table_location: String,
    schema: SchemaRef,
    equality_ids: Vec<i32>,
}

impl EqualityDeleteWriter {
    pub fn new(
        storage: Storage,
        table_location: String,
        schema: SchemaRef,
        equality_ids: Vec<i32>,
    ) -> Self {
        Self {
            storage,
            table_location,
            schema,
            equality_ids,
        }
    }

    /// Writes a batch of equality deletes.
    pub async fn write_batch(
        &self,
        batch: &RecordBatch,
        file_id: &str,
    ) -> anyhow::Result<DataFile> {
        let props = WriterProperties::builder().build();
        let mut buffer = Cursor::new(Vec::new());
        {
            let mut writer = ArrowWriter::try_new(&mut buffer, self.schema.clone(), Some(props))?;
            writer.write(batch)?;
            writer.close()?;
        }

        let data = buffer.into_inner();
        let file_size = data.len() as i64;
        let record_count = batch.num_rows() as i64;
        let path = format!("{}/data/{}-eq-delete.parquet", self.table_location, file_id);

        self.storage.write(&path, Bytes::from(data)).await?;

        // Calculate stats for pruning
        let stats = crate::statistics::calculate_stats(batch)?;

        let mut data_file = DataFile::new(path, FileFormat::Parquet, record_count, file_size)
            .with_content(FileContent::EqualityDeletes)
            .with_equality_ids(self.equality_ids.clone());

        data_file.statistics = stats;

        Ok(data_file)
    }
}