hudi-core 0.2.0

A native Rust library for Apache Hudi
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
use crate::config::table::HudiTableConfig;
use crate::config::utils::split_hudi_options_from_others;
use crate::config::HudiConfigs;
use crate::file_group::FileSlice;
use crate::storage::Storage;
use anyhow::Result;
use arrow_array::RecordBatch;
use std::sync::Arc;

/// File group reader handles all read operations against a file group.
#[derive(Clone, Debug)]
pub struct FileGroupReader {
    storage: Arc<Storage>,
}

impl FileGroupReader {
    pub fn new(storage: Arc<Storage>) -> Self {
        Self { storage }
    }

    pub fn new_with_options<I, K, V>(base_uri: &str, options: I) -> Result<Self>
    where
        I: IntoIterator<Item = (K, V)>,
        K: AsRef<str>,
        V: Into<String>,
    {
        let (mut hudi_opts, others) = split_hudi_options_from_others(options);
        hudi_opts.insert(
            HudiTableConfig::BasePath.as_ref().to_string(),
            base_uri.to_string(),
        );

        let hudi_configs = Arc::new(HudiConfigs::new(hudi_opts));

        let storage = Storage::new(Arc::new(others), hudi_configs)?;
        Ok(Self { storage })
    }

    pub async fn read_file_slice_by_base_file_path(
        &self,
        relative_path: &str,
    ) -> Result<RecordBatch> {
        self.storage.get_parquet_file_data(relative_path).await
    }

    pub async fn read_file_slice(&self, file_slice: &FileSlice) -> Result<RecordBatch> {
        self.read_file_slice_by_base_file_path(&file_slice.base_file_relative_path())
            .await
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use url::Url;

    #[test]
    fn test_new() {
        let base_url = Url::parse("file:///tmp/hudi_data").unwrap();
        let storage = Storage::new_with_base_url(base_url).unwrap();
        let fg_reader = FileGroupReader::new(storage.clone());
        assert!(Arc::ptr_eq(&fg_reader.storage, &storage));
    }

    #[test]
    fn test_new_with_options() -> Result<()> {
        let options = vec![("key1", "value1"), ("key2", "value2")];
        let reader = FileGroupReader::new_with_options("/tmp/hudi_data", options)?;
        assert!(!reader.storage.options.is_empty());
        assert!(reader
            .storage
            .hudi_configs
            .contains(HudiTableConfig::BasePath));
        Ok(())
    }
}