icepick 0.4.1

Experimental Rust client for Apache Iceberg with WASM support for AWS S3 Tables and Cloudflare R2
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! Manifest file reading

use super::parse::{parse_manifest_entry, parse_manifest_entry_with_stats};
use super::{DataFileEntry, DataFileStats};
use crate::error::{Error, Result};
use crate::io::FileIO;
use apache_avro::types::Value;
use apache_avro::Reader as AvroReader;

/// Reads manifest files
pub struct ManifestReader;

impl ManifestReader {
    /// Read a manifest and return data file entries (excluding deleted files)
    pub async fn read(file_io: &FileIO, manifest_path: &str) -> Result<Vec<DataFileEntry>> {
        let bytes = file_io.read(manifest_path).await?;

        let reader = AvroReader::new(&bytes[..])
            .map_err(|e| Error::invalid_input(format!("Failed to read manifest: {}", e)))?;

        let mut data_files = Vec::new();

        for (idx, value) in reader.enumerate() {
            let value = value.map_err(|e| {
                Error::invalid_input(format!("Failed to parse manifest entry {}: {}", idx, e))
            })?;

            if let Value::Record(fields) = value {
                if let Some(entry) = parse_manifest_entry(fields).map_err(|e| {
                    Error::invalid_input(format!("Invalid manifest entry {}: {}", idx, e))
                })? {
                    data_files.push(entry);
                }
            }
        }

        Ok(data_files)
    }

    /// Read a manifest and return data file entries with full statistics for pruning
    pub async fn read_with_stats(
        file_io: &FileIO,
        manifest_path: &str,
    ) -> Result<Vec<DataFileStats>> {
        let bytes = file_io.read(manifest_path).await?;

        let reader = AvroReader::new(&bytes[..])
            .map_err(|e| Error::invalid_input(format!("Failed to read manifest: {}", e)))?;

        let mut data_files = Vec::new();

        for (idx, value) in reader.enumerate() {
            let value = value.map_err(|e| {
                Error::invalid_input(format!("Failed to parse manifest entry {}: {}", idx, e))
            })?;

            if let Value::Record(fields) = value {
                let entry_opt = parse_manifest_entry_with_stats(fields).map_err(|e| {
                    Error::invalid_input(format!("Invalid manifest entry {}: {}", idx, e))
                })?;
                if let Some(entry) = entry_opt {
                    data_files.push(entry);
                }
            }
        }

        Ok(data_files)
    }
}