arrow2 0.18.0

Unofficial implementation of Apache Arrow spec in safe Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

use avro_schema::schema::{Enum, Fixed, Record, Schema as AvroSchema};

use crate::datatypes::*;
use crate::error::{Error, Result};

fn external_props(schema: &AvroSchema) -> Metadata {
    let mut props = Metadata::new();
    match &schema {
        AvroSchema::Record(Record {
            doc: Some(ref doc), ..
        })
        | AvroSchema::Enum(Enum {
            doc: Some(ref doc), ..
        }) => {
            props.insert("avro::doc".to_string(), doc.clone());
        }
        _ => {}
    }
    props
}

/// Infers an [`Schema`] from the root [`Record`].
/// This
pub fn infer_schema(record: &Record) -> Result<Schema> {
    Ok(record
        .fields
        .iter()
        .map(|field| {
            schema_to_field(
                &field.schema,
                Some(&field.name),
                external_props(&field.schema),
            )
        })
        .collect::<Result<Vec<_>>>()?
        .into())
}

fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> Result<Field> {
    let mut nullable = false;
    let data_type = match schema {
        AvroSchema::Null => DataType::Null,
        AvroSchema::Boolean => DataType::Boolean,
        AvroSchema::Int(logical) => match logical {
            Some(logical) => match logical {
                avro_schema::schema::IntLogical::Date => DataType::Date32,
                avro_schema::schema::IntLogical::Time => DataType::Time32(TimeUnit::Millisecond),
            },
            None => DataType::Int32,
        },
        AvroSchema::Long(logical) => match logical {
            Some(logical) => match logical {
                avro_schema::schema::LongLogical::Time => DataType::Time64(TimeUnit::Microsecond),
                avro_schema::schema::LongLogical::TimestampMillis => {
                    DataType::Timestamp(TimeUnit::Millisecond, Some("00:00".to_string()))
                }
                avro_schema::schema::LongLogical::TimestampMicros => {
                    DataType::Timestamp(TimeUnit::Microsecond, Some("00:00".to_string()))
                }
                avro_schema::schema::LongLogical::LocalTimestampMillis => {
                    DataType::Timestamp(TimeUnit::Millisecond, None)
                }
                avro_schema::schema::LongLogical::LocalTimestampMicros => {
                    DataType::Timestamp(TimeUnit::Microsecond, None)
                }
            },
            None => DataType::Int64,
        },
        AvroSchema::Float => DataType::Float32,
        AvroSchema::Double => DataType::Float64,
        AvroSchema::Bytes(logical) => match logical {
            Some(logical) => match logical {
                avro_schema::schema::BytesLogical::Decimal(precision, scale) => {
                    DataType::Decimal(*precision, *scale)
                }
            },
            None => DataType::Binary,
        },
        AvroSchema::String(_) => DataType::Utf8,
        AvroSchema::Array(item_schema) => DataType::List(Box::new(schema_to_field(
            item_schema,
            Some("item"), // default name for list items
            Metadata::default(),
        )?)),
        AvroSchema::Map(_) => todo!("Avro maps are mapped to MapArrays"),
        AvroSchema::Union(schemas) => {
            // If there are only two variants and one of them is null, set the other type as the field data type
            let has_nullable = schemas.iter().any(|x| x == &AvroSchema::Null);
            if has_nullable && schemas.len() == 2 {
                nullable = true;
                if let Some(schema) = schemas
                    .iter()
                    .find(|&schema| !matches!(schema, AvroSchema::Null))
                {
                    schema_to_field(schema, None, Metadata::default())?.data_type
                } else {
                    return Err(Error::NotYetImplemented(format!(
                        "Can't read avro union {schema:?}"
                    )));
                }
            } else {
                let fields = schemas
                    .iter()
                    .map(|s| schema_to_field(s, None, Metadata::default()))
                    .collect::<Result<Vec<Field>>>()?;
                DataType::Union(fields, None, UnionMode::Dense)
            }
        }
        AvroSchema::Record(Record { fields, .. }) => {
            let fields = fields
                .iter()
                .map(|field| {
                    let mut props = Metadata::new();
                    if let Some(doc) = &field.doc {
                        props.insert("avro::doc".to_string(), doc.clone());
                    }
                    schema_to_field(&field.schema, Some(&field.name), props)
                })
                .collect::<Result<_>>()?;
            DataType::Struct(fields)
        }
        AvroSchema::Enum { .. } => {
            return Ok(Field::new(
                name.unwrap_or_default(),
                DataType::Dictionary(IntegerType::Int32, Box::new(DataType::Utf8), false),
                false,
            ))
        }
        AvroSchema::Fixed(Fixed { size, logical, .. }) => match logical {
            Some(logical) => match logical {
                avro_schema::schema::FixedLogical::Decimal(precision, scale) => {
                    DataType::Decimal(*precision, *scale)
                }
                avro_schema::schema::FixedLogical::Duration => {
                    DataType::Interval(IntervalUnit::MonthDayNano)
                }
            },
            None => DataType::FixedSizeBinary(*size),
        },
    };

    let name = name.unwrap_or_default();

    Ok(Field::new(name, data_type, nullable).with_metadata(props))
}