myval 0.1.22

Lightweight Apache Arrow data frame
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

#[cfg(feature = "arrow2_ih")]
extern crate arrow2_ih as arrow2;

use crate::df::{DataFrame, Series};
use crate::Error;
use arrow2::array::{new_null_array, Array};
use arrow2::compute;
use arrow2::datatypes::{DataType, Field};
use std::collections::BTreeMap;

enum El<'a> {
    Array(&'a dyn Array),
    Null(Option<Box<dyn Array>>),
}

struct ArrInfo {
    dtype: DataType,
    rows: usize,
}

/// concat multiple data frames
///
/// # Panics
///
/// Should not panic
pub fn concat(data_frames: &[&DataFrame]) -> Result<DataFrame, Error> {
    if data_frames.is_empty() {
        Ok(DataFrame::new0())
    } else {
        let mut fields: Vec<Field> = Vec::new();
        let mut meta: BTreeMap<String, String> = BTreeMap::new();
        // collect all possible fields
        for df in data_frames {
            for (n, v) in df.metadata() {
                if !meta.contains_key(n) {
                    meta.insert(n.clone(), v.clone());
                }
            }
            if !df.is_empty() {
                for field in df.fields() {
                    if !fields.iter().any(|f| f.name == field.name) {
                        fields.push(field.clone());
                    }
                }
            }
        }
        let mut data: Vec<Series> = Vec::with_capacity(fields.len());
        // walk thru all fields in all data frames
        for field in &fields {
            let mut serie: Vec<El> = Vec::with_capacity(data_frames.len());
            for df in data_frames {
                if !df.is_empty() {
                    if let Some((s, _)) = df.get_series(&field.name) {
                        serie.push(El::Array(s.as_ref()));
                    } else {
                        serie.push(El::Null(None));
                    }
                }
            }
            let mut info = None;
            for s in &serie {
                if let El::Array(x) = s {
                    info.replace(ArrInfo {
                        dtype: x.data_type().clone(),
                        rows: x.len(),
                    });
                    break;
                }
            }
            if let Some(i) = info {
                let mut serie_data = Vec::with_capacity(serie.len());
                for s in &mut serie {
                    match s {
                        El::Array(a) => serie_data.push(*a),
                        El::Null(h) => {
                            let arr = new_null_array(i.dtype.clone(), i.rows).to_boxed();
                            h.replace(arr);
                            serie_data.push(h.as_ref().unwrap().as_ref());
                        }
                    }
                }
                let c_data = compute::concatenate::concatenate(&serie_data)?;
                data.push(c_data);
            }
        }
        DataFrame::from_parts(fields, data, Some(meta))
    }
}