trs-dataframe 0.10.1

Dataframe library for Teiresias
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

pub mod candidate;
pub mod dataframe;
pub mod error;
pub use candidate::CandidateData;
pub use data_value;
pub use data_value::DataValue;
pub use dataframe::DataFrame;
pub mod filter;
pub use dataframe::join::{JoinBy, JoinById, JoinRelation};
pub use dataframe::{
    column_store::{ColumnFrame, KeyIndex},
    index::hash_datavalue,
    key::Key,
};
pub type MLChefMap = halfbrown::HashMap<smartstring::alias::String, Vec<DataValue>>;
pub use ndarray;

#[cfg(feature = "jmalloc")]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

#[cfg(feature = "polars-df")]
pub use polars;
/// Data type for the values in the dataframe
#[derive(
    Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash, Default,
)]
#[cfg_attr(feature = "python", pyo3::pyclass(eq, eq_int))]
#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
pub enum DataType {
    Bool,
    U32,
    I32,
    U8,
    U64,
    I64,
    F32,
    F64,
    I128,
    U128,
    String,
    Bytes,
    #[default]
    Unknown,
    Vec,
    Map,
}

/// Autodetector for the data type from [`DataValue`]
pub fn detect_dtype(value: &DataValue) -> DataType {
    use DataValue::*;
    match value {
        Bool(_) => DataType::Bool,
        I32(_) => DataType::I32,
        U32(_) => DataType::U32,
        I64(_) => DataType::I64,
        U64(_) => DataType::U64,
        F32(_) => DataType::F32,
        F64(_) => DataType::F64,
        I128(_) => DataType::I128,
        U128(_) => DataType::U128,
        String(_) => DataType::String,
        Bytes(_) => DataType::Bytes,
        Vec(_) => DataType::Vec,
        Map(_) => DataType::Map,
        _ => DataType::Unknown,
    }
}
pub fn detect_dtype_arr(value: &[DataValue]) -> DataType {
    let mut dtype = DataType::Unknown;
    let mut find_count = 3;
    for val in value {
        let new_dtype = detect_dtype(val);
        if new_dtype != dtype {
            dtype = new_dtype;
        } else if new_dtype == dtype && !matches!(dtype, DataType::Unknown) {
            find_count -= 1;
        }
        if find_count == 0 {
            break;
        }
    }

    dtype
}

#[cfg(feature = "python")]
use pyo3::prelude::*;

#[cfg(feature = "python")]
///
/// ```
/// use pyo3::prelude::*;
///
///  fn main() {
///     let result = pyo3::Python::with_gil(|py| -> PyResult<()> {
///         let module = PyModule::new(py, "trs_dataframe")?;
///         let _m = trs_dataframe::trs_dataframe(py, module)?;
///         Ok(())
///         });
///     assert!(result.is_ok(), "{:?}", result);
///  }
///
/// ```
#[pymodule]
pub fn trs_dataframe(_py: pyo3::Python<'_>, m: pyo3::Bound<'_, PyModule>) -> pyo3::PyResult<()> {
    m.add_class::<DataFrame>()?;
    m.add_class::<JoinRelation>()?;
    m.add_class::<Key>()?;
    Ok(())
}
#[cfg(test)]
mod test {
    use crate::dataframe::column_store::convert_data_value;

    use super::*;
    use rstest::*;

    #[rstest]
    #[case(DataType::Bool, DataValue::Bool(true))]
    #[case(DataType::I32, DataValue::I32(1))]
    #[case(DataType::U32, DataValue::U32(1))]
    #[case(DataType::I64, DataValue::I64(1))]
    #[case(DataType::U64, DataValue::U64(1))]
    #[case(DataType::F32, DataValue::F32(1.0))]
    #[case(DataType::F64, DataValue::F64(1.0))]
    #[case(DataType::U128, DataValue::U128(1))]
    #[case(DataType::I128, DataValue::I128(1))]
    #[case(DataType::String, DataValue::String("1".into()))]
    #[case(DataType::Bytes, DataValue::Bytes(b"1".to_vec()))]
    #[case(DataType::Vec, DataValue::Vec(vec![DataValue::I32(1)]))]
    #[case(DataType::Map, DataValue::Map(std::collections::HashMap::new()))]
    #[case(DataType::Unknown, DataValue::Null)]
    fn detection_test(#[case] dtype: DataType, #[case] value: DataValue) {
        assert_eq!(detect_dtype(&value), dtype);
        let serde_dtype: DataType =
            serde_json::from_str(&serde_json::to_string(&dtype).expect("BUG: cannot serialize"))
                .expect("BUG: cannot deserialize");
        assert_eq!(serde_dtype, dtype);
        let dt = convert_data_value(value.clone(), dtype);
        assert_eq!(dt, value);
    }
}