use std::collections::VecDeque;
use std::marker::PhantomData;
#[cfg(feature = "serialize")]
use serde::ser::{Serialize, SerializeSeq, Serializer};
use std::fmt::Debug;
use std::rc::Rc;
use std::sync::Arc;
use typenum::UTerm;
use access::{DataIndex, NRows};
use cons::Nil;
use error;
use field::{FieldData, Value};
use fieldlist::FieldCons;
use label::*;
use permute::{self, UpdatePermutation};
use select::{FieldSelect, SelectFieldByLabel};
use store::{AssocFrameLookup, AssocStorage, DataRef, DataStore, IntoView};
use view::{DataView, ViewFrameCons};
type Permutation = permute::Permutation<Vec<usize>>;
pub type StoreFieldCons<L, T> = LCons<L, T>;
pub struct StoreFieldMarkers<FrameType, StoreFieldList> {
_marker: PhantomData<(FrameType, StoreFieldList)>,
}
pub type FieldLookupCons<FrameLabel, StoreDetails, Tail> = LMCons<FrameLabel, StoreDetails, Tail>;
pub struct Single;
pub struct Melt;
pub trait SimpleFrameFields {
type Fields;
}
impl SimpleFrameFields for Nil {
type Fields = Nil;
}
impl<Label, Value, Tail> SimpleFrameFields for LVCons<Label, Value, Tail>
where
Tail: SimpleFrameFields,
{
type Fields = FieldLookupCons<
Label,
StoreFieldMarkers<Single, Labels![Label]>,
<Tail as SimpleFrameFields>::Fields,
>;
}
pub trait IntoStrFrame<Label> {
type Output;
fn into_str_frame() -> Self::Output;
fn into_repeated_str_frame(n: usize) -> Self::Output;
}
impl<Label, List> IntoStrFrame<Label> for List
where
Label: Debug,
List: StrLabels,
{
type Output = DataFrame<
<FieldCons<Label, String, Nil> as SimpleFrameFields>::Fields,
DataStore<FieldCons<Label, String, Nil>>,
>;
fn into_str_frame() -> Self::Output {
let strs: FieldData<String> = Self::labels().iter().map(|&s| s.to_owned()).collect();
DataFrame::from(DataStore::<Nil>::empty().push_back_field::<Label, _>(strs))
}
fn into_repeated_str_frame(n: usize) -> Self::Output {
let strs: FieldData<String> = Self::labels().iter().map(|&s| s.to_owned()).collect();
DataFrame::from_repeated_store(
DataStore::<Nil>::empty().push_back_field::<Label, _>(strs),
n,
)
}
}
pub trait MeltFrameFields<MeltLabel> {
type Fields;
}
impl<MeltLabel, Labels> MeltFrameFields<MeltLabel> for Labels
where
Labels: AssocLabels,
{
type Fields =
FieldLookupCons<MeltLabel, StoreFieldMarkers<Melt, <Labels as AssocLabels>::Labels>, Nil>;
}
#[derive(Debug)]
pub struct DataFrame<FrameFields, FramedStore> {
permutation: Rc<Permutation>,
fields: PhantomData<FrameFields>,
store: Arc<FramedStore>,
}
impl<FrameFields, FramedStore> DataFrame<FrameFields, FramedStore>
where
FramedStore: NRows,
{
pub fn len(&self) -> usize {
match self.permutation.len() {
Some(len) => len,
None => self.store.nrows(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
impl<FrameFields, FramedStore> NRows for DataFrame<FrameFields, FramedStore>
where
FramedStore: NRows,
{
fn nrows(&self) -> usize {
self.len()
}
}
impl<FrameFields, FramedStore> Clone for DataFrame<FrameFields, FramedStore> {
fn clone(&self) -> DataFrame<FrameFields, FramedStore> {
DataFrame {
permutation: self.permutation.clone(),
fields: PhantomData,
store: Arc::clone(&self.store),
}
}
}
#[cfg(test)]
pub trait StoreRefCount {
fn store_ref_count(&self) -> usize;
}
#[cfg(test)]
impl<FrameFields, FramedStore> StoreRefCount for DataFrame<FrameFields, FramedStore> {
fn store_ref_count(&self) -> usize {
Arc::strong_count(&self.store)
}
}
impl<FrameFields, FramedStore> UpdatePermutation for DataFrame<FrameFields, FramedStore> {
fn update_permutation(mut self, new_permutation: &[usize]) -> Self {
let perm = (*self.permutation).clone();
self.permutation = Rc::new(perm.update_indices(new_permutation));
self
}
}
impl<StoreFields> From<DataStore<StoreFields>>
for DataFrame<<StoreFields as SimpleFrameFields>::Fields, DataStore<StoreFields>>
where
StoreFields: AssocStorage + SimpleFrameFields,
{
fn from(
store: DataStore<StoreFields>,
) -> DataFrame<<StoreFields as SimpleFrameFields>::Fields, DataStore<StoreFields>> {
DataFrame {
permutation: Rc::new(Permutation::default()),
fields: PhantomData,
store: Arc::new(store),
}
}
}
impl<Labels, Frames> From<DataView<Labels, Frames>>
for DataFrame<<Labels as SimpleFrameFields>::Fields, DataView<Labels, Frames>>
where
Labels: SimpleFrameFields,
{
fn from(
view: DataView<Labels, Frames>,
) -> DataFrame<<Labels as SimpleFrameFields>::Fields, DataView<Labels, Frames>> {
DataFrame {
permutation: Rc::new(Permutation::default()),
fields: PhantomData,
store: Arc::new(view),
}
}
}
pub trait IntoFrame {
type FrameFields;
type FramedStore;
type Output;
fn into_frame(self) -> Self::Output;
}
impl<Labels, Frames> IntoFrame for DataView<Labels, Frames>
where
Labels: SimpleFrameFields,
DataFrame<<Labels as SimpleFrameFields>::Fields, DataView<Labels, Frames>>:
From<DataView<Labels, Frames>>,
{
type FrameFields = <Labels as SimpleFrameFields>::Fields;
type FramedStore = DataView<Labels, Frames>;
type Output = DataFrame<Self::FrameFields, Self::FramedStore>;
fn into_frame(self) -> Self::Output {
self.into()
}
}
impl<Fields> IntoFrame for DataStore<Fields>
where
Fields: AssocStorage + SimpleFrameFields,
DataFrame<<Fields as SimpleFrameFields>::Fields, DataStore<Fields>>: From<DataStore<Fields>>,
{
type FrameFields = <Fields as SimpleFrameFields>::Fields;
type FramedStore = DataStore<Fields>;
type Output = DataFrame<Self::FrameFields, Self::FramedStore>;
fn into_frame(self) -> Self::Output {
self.into()
}
}
pub trait IntoMeltFrame<MeltLabel> {
type FrameFields;
type FramedStore;
type Output;
fn into_melt_frame(self) -> Self::Output;
}
impl<MeltLabel, Labels, Frames> IntoMeltFrame<MeltLabel> for DataView<Labels, Frames>
where
Labels: MeltFrameFields<MeltLabel>,
{
type FrameFields = <Labels as MeltFrameFields<MeltLabel>>::Fields;
type FramedStore = DataView<Labels, Frames>;
type Output = DataFrame<Self::FrameFields, Self::FramedStore>;
fn into_melt_frame(self) -> Self::Output {
DataFrame {
permutation: Rc::new(Permutation::default()),
fields: PhantomData,
store: Arc::new(self),
}
}
}
impl<FrameFields, FramedStore> IntoView for DataFrame<FrameFields, FramedStore>
where
FrameFields: AssocFrameLookup,
{
type Labels = <FrameFields as AssocFrameLookup>::Output;
type Frames = ViewFrameCons<UTerm, Self, Nil>;
type Output = DataView<Self::Labels, Self::Frames>;
fn into_view(self) -> Self::Output {
DataView::new(ViewFrameCons {
head: self.into(),
tail: Nil,
})
}
}
impl<StoreFields> DataFrame<<StoreFields as SimpleFrameFields>::Fields, DataStore<StoreFields>>
where
StoreFields: AssocStorage + SimpleFrameFields,
DataStore<StoreFields>: NRows,
{
pub(crate) fn from_repeated_store(
store: DataStore<StoreFields>,
reps: usize,
) -> DataFrame<<StoreFields as SimpleFrameFields>::Fields, DataStore<StoreFields>> {
DataFrame {
permutation: {
let mut v = Vec::with_capacity(store.nrows() * reps);
for _ in 0..reps {
v.extend(0..store.nrows());
}
Rc::new(v.into())
},
fields: PhantomData,
store: Arc::new(store),
}
}
}
impl<FrameFields, FramedStore> SelfValued for DataFrame<FrameFields, FramedStore> {}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
enum FrameKind<DI> {
Single(DI),
Melt(Vec<DI>),
}
impl<DI> FrameKind<DI>
where
DI: DataIndex,
{
fn nfields(&self) -> usize {
match *self {
FrameKind::Single(_) => 1,
FrameKind::Melt(ref fields) => fields.len(),
}
}
fn nrows(&self) -> usize {
assert!(!self.is_empty());
match *self {
FrameKind::Single(ref field) => field.len(),
FrameKind::Melt(ref fields) => fields[0].len(),
}
}
fn is_empty(&self) -> bool {
self.nfields() == 0
}
}
#[derive(Debug, Hash, PartialEq, Eq)]
pub struct Framed<T, DI> {
permutation: Rc<Permutation>,
data: FrameKind<DI>,
_ty: PhantomData<T>,
}
impl<T, DI> Framed<T, DI> {
pub fn new(permutation: Rc<Permutation>, data: DI) -> Framed<T, DI> {
Framed {
permutation,
data: FrameKind::Single(data),
_ty: PhantomData,
}
}
pub fn new_melt(permutation: Rc<Permutation>, data: Vec<DI>) -> Framed<T, DI> {
Framed {
permutation,
data: FrameKind::Melt(data),
_ty: PhantomData,
}
}
}
impl<T, DI> Clone for Framed<T, DI>
where
DI: Clone,
{
fn clone(&self) -> Framed<T, DI> {
Framed {
permutation: Rc::clone(&self.permutation),
data: self.data.clone(),
_ty: PhantomData,
}
}
}
impl<T> From<DataRef<T>> for Framed<T, DataRef<T>> {
fn from(orig: DataRef<T>) -> Framed<T, DataRef<T>> {
Framed {
permutation: Rc::new(Permutation::default()),
data: FrameKind::Single(orig),
_ty: PhantomData,
}
}
}
impl<T> From<FieldData<T>> for Framed<T, DataRef<T>> {
fn from(orig: FieldData<T>) -> Framed<T, DataRef<T>> {
Framed {
permutation: Rc::new(Permutation::default()),
data: FrameKind::Single(orig.into()),
_ty: PhantomData,
}
}
}
impl<T, DI> DataIndex for Framed<T, DI>
where
T: Debug,
DI: DataIndex<DType = T> + Debug,
{
type DType = T;
fn get_datum(&self, idx: usize) -> error::Result<Value<&T>> {
assert!(!self.data.is_empty());
match self.data {
FrameKind::Single(ref field) => field.get_datum(self.permutation.map_index(idx)),
FrameKind::Melt(ref fields) => {
let nfields = self.data.nfields();
fields[idx % nfields].get_datum(self.permutation.map_index(idx / nfields))
}
}
}
fn len(&self) -> usize {
assert!(!self.data.is_empty());
self.data.nfields() * self.permutation.len().unwrap_or(self.data.nrows())
}
}
#[cfg(feature = "serialize")]
impl<T, DI> Serialize for Framed<T, DI>
where
T: Serialize,
Self: DataIndex<DType = T>,
{
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.len()))?;
for elem in self.iter() {
seq.serialize_element(&elem)?;
}
seq.end()
}
}
pub trait SelectAndFrame<Label, FramedStore> {
type DType: Debug;
type Field: DataIndex<DType = Self::DType>;
fn select_and_frame(
perm: &Rc<Permutation>,
store: &FramedStore,
) -> Framed<Self::DType, Self::Field>;
}
pub trait SelectAndFrameMatch<Label, FramedStore, Match> {
type DType: Debug;
type Field: DataIndex<DType = Self::DType>;
fn select_and_frame(
perm: &Rc<Permutation>,
store: &FramedStore,
) -> Framed<Self::DType, Self::Field>;
}
impl<TargetLabel, FrameLabel, StoreDetails, Tail, FramedStore>
SelectAndFrame<TargetLabel, FramedStore> for FieldLookupCons<FrameLabel, StoreDetails, Tail>
where
TargetLabel: LabelEq<FrameLabel>,
FieldLookupCons<FrameLabel, StoreDetails, Tail>:
SelectAndFrameMatch<TargetLabel, FramedStore, <TargetLabel as LabelEq<FrameLabel>>::Eq>,
{
type DType = <FieldLookupCons<FrameLabel, StoreDetails, Tail> as SelectAndFrameMatch<
TargetLabel,
FramedStore,
<TargetLabel as LabelEq<FrameLabel>>::Eq,
>>::DType;
type Field = <FieldLookupCons<FrameLabel, StoreDetails, Tail> as SelectAndFrameMatch<
TargetLabel,
FramedStore,
<TargetLabel as LabelEq<FrameLabel>>::Eq,
>>::Field;
fn select_and_frame(
perm: &Rc<Permutation>,
store: &FramedStore,
) -> Framed<Self::DType, Self::Field> {
<Self as SelectAndFrameMatch<
TargetLabel,
FramedStore,
<TargetLabel as LabelEq<FrameLabel>>::Eq,
>>::select_and_frame(perm, store)
}
}
impl<TargetLabel, FrameLabel, StoreFieldList, Tail, FramedStore>
SelectAndFrameMatch<TargetLabel, FramedStore, True>
for FieldLookupCons<FrameLabel, StoreFieldMarkers<Single, StoreFieldList>, Tail>
where
FramedStore: SelectFieldByLabel<TargetLabel>,
<FramedStore as SelectFieldByLabel<TargetLabel>>::DType: Debug,
{
type DType = <FramedStore as SelectFieldByLabel<TargetLabel>>::DType;
type Field = <FramedStore as SelectFieldByLabel<TargetLabel>>::Output;
fn select_and_frame(
perm: &Rc<Permutation>,
store: &FramedStore,
) -> Framed<Self::DType, Self::Field> {
Framed::new(
Rc::clone(perm),
SelectFieldByLabel::<TargetLabel>::select_field(store),
)
}
}
impl<TargetLabel, FrameLabel, StoreFieldList, Tail, FramedStore>
SelectAndFrameMatch<TargetLabel, FramedStore, True>
for FieldLookupCons<FrameLabel, StoreFieldMarkers<Melt, StoreFieldList>, Tail>
where
StoreFieldList: RotateFields<FramedStore>,
<StoreFieldList as RotateFields<FramedStore>>::DType: Debug,
<StoreFieldList as RotateFields<FramedStore>>::Output: Clone,
{
type DType = <StoreFieldList as RotateFields<FramedStore>>::DType;
type Field = <StoreFieldList as RotateFields<FramedStore>>::Output;
fn select_and_frame(
perm: &Rc<Permutation>,
store: &FramedStore,
) -> Framed<Self::DType, Self::Field> {
let melt_rotation = <StoreFieldList as RotateFields<FramedStore>>::add_to_rotation(store);
Framed::new_melt(
Rc::clone(perm),
melt_rotation.iter().cloned().collect::<Vec<_>>(),
)
}
}
impl<TargetLabel, FrameLabel, StoreDetails, Tail, FramedStore>
SelectAndFrameMatch<TargetLabel, FramedStore, False>
for FieldLookupCons<FrameLabel, StoreDetails, Tail>
where
Tail: SelectAndFrame<TargetLabel, FramedStore>,
{
type DType = <Tail as SelectAndFrame<TargetLabel, FramedStore>>::DType;
type Field = <Tail as SelectAndFrame<TargetLabel, FramedStore>>::Field;
fn select_and_frame(
perm: &Rc<Permutation>,
store: &FramedStore,
) -> Framed<Self::DType, Self::Field> {
<Tail as SelectAndFrame<TargetLabel, FramedStore>>::select_and_frame(perm, store)
}
}
pub trait RotateFields<FramedStore> {
type DType;
type Output: DataIndex<DType = Self::DType>;
fn add_to_rotation(store: &FramedStore) -> VecDeque<Self::Output>;
}
impl<FramedStore> RotateFields<FramedStore> for Nil {
type DType = usize;
type Output = DataRef<usize>;
fn add_to_rotation(_store: &FramedStore) -> VecDeque<Self::Output> {
VecDeque::new()
}
}
impl<FramedStore, Label, Tail> RotateFields<FramedStore> for StoreFieldCons<Label, Tail>
where
FramedStore: SelectFieldByLabel<Label>,
Tail: RotateFieldsTerm<
FramedStore,
<FramedStore as SelectFieldByLabel<Label>>::DType,
<FramedStore as SelectFieldByLabel<Label>>::Output,
>,
{
type DType = <FramedStore as SelectFieldByLabel<Label>>::DType;
type Output = <FramedStore as SelectFieldByLabel<Label>>::Output;
fn add_to_rotation(store: &FramedStore) -> VecDeque<Self::Output> {
let mut v = Tail::add_to_rotation(store);
v.push_front(SelectFieldByLabel::<Label>::select_field(store));
v
}
}
pub trait RotateFieldsTerm<FramedStore, DType, Output> {
fn add_to_rotation(store: &FramedStore) -> VecDeque<Output>;
}
impl<FramedStore, DType, Output> RotateFieldsTerm<FramedStore, DType, Output> for Nil {
fn add_to_rotation(_store: &FramedStore) -> VecDeque<Output> {
VecDeque::new()
}
}
impl<FramedStore, DType, Output, Label, Tail> RotateFieldsTerm<FramedStore, DType, Output>
for StoreFieldCons<Label, Tail>
where
StoreFieldCons<Label, Tail>: RotateFields<FramedStore, DType = DType, Output = Output>,
Output: DataIndex<DType = DType>,
{
fn add_to_rotation(store: &FramedStore) -> VecDeque<Output> {
<StoreFieldCons<Label, Tail> as RotateFields<FramedStore>>::add_to_rotation(store)
}
}
impl<FrameFields, FramedStore, Label> SelectFieldByLabel<Label>
for DataFrame<FrameFields, FramedStore>
where
FrameFields: SelectAndFrame<Label, FramedStore>,
{
type DType = <FrameFields as SelectAndFrame<Label, FramedStore>>::DType;
type Output = Framed<Self::DType, <FrameFields as SelectAndFrame<Label, FramedStore>>::Field>;
fn select_field(&self) -> Self::Output {
<FrameFields as SelectAndFrame<Label, FramedStore>>::select_and_frame(
&self.permutation,
&*self.store,
)
}
}
impl<FrameFields, FramedStore> FieldSelect for DataFrame<FrameFields, FramedStore> {}
#[cfg(test)]
mod tests {
use std::path::Path;
use csv_sniffer::metadata::Metadata;
use serde_json;
use super::*;
use source::csv::{CsvReader, CsvSource, IntoCsvSrcSchema};
fn load_csv_file<Schema>(
filename: &str,
schema: Schema,
) -> (CsvReader<Schema::CsvSrcSchema>, Metadata)
where
Schema: IntoCsvSrcSchema,
<Schema as IntoCsvSrcSchema>::CsvSrcSchema: Debug,
{
let data_filepath = Path::new(file!())
.parent()
.unwrap()
.parent()
.unwrap()
.join("tests")
.join("data")
.join(filename);
let source = CsvSource::new(data_filepath).unwrap();
(
CsvReader::new(&source, schema).unwrap(),
source.metadata().clone(),
)
}
tablespace![
pub table gdp {
CountryName: String,
CountryCode: String,
Year1983: f64
}
];
#[test]
fn frame_select() {
let gdp_schema = schema![
fieldname gdp::CountryName = "Country Name";
fieldname gdp::CountryCode = "Country Code";
fieldname gdp::Year1983 = "1983";
];
let (mut csv_rdr, _metadata) = load_csv_file("gdp.csv", gdp_schema.clone());
let ds = csv_rdr.read().unwrap();
let frame = DataFrame::from(ds);
println!("{:?}", frame.field::<gdp::CountryName>());
}
#[test]
fn framed_serialize() {
let field: FieldData<f64> = vec![5.0f64, 3.4, -1.3, 5.2, 6.0, -126.9].into();
let framed: Framed<f64, _> = field.into();
assert_eq!(
serde_json::to_string(&framed).unwrap(),
"[5.0,3.4,-1.3,5.2,6.0,-126.9]"
);
println!("{}", serde_json::to_string(&framed).unwrap());
}
tablespace![
pub table order {
Name: String,
Name1: String,
Name2: String,
Name3: String,
}
];
#[test]
fn repeated() {
let field: FieldData<String> = vec!["First", "Second", "Third"]
.iter()
.map(|&s| s.to_owned())
.collect();
let store = DataStore::<Nil>::empty().push_back_field::<order::Name, _>(field);
let frame = DataFrame::from_repeated_store(store, 5);
assert_eq!(
frame.field::<order::Name>().to_vec(),
vec![
"First", "Second", "Third", "First", "Second", "Third", "First", "Second", "Third",
"First", "Second", "Third", "First", "Second", "Third",
]
);
let dv = frame.into_view();
println!("{}", dv);
assert_eq!(
dv.field::<order::Name>().to_vec(),
vec![
"First", "Second", "Third", "First", "Second", "Third", "First", "Second", "Third",
"First", "Second", "Third", "First", "Second", "Third",
]
);
}
#[test]
fn framed_melt() {
let store = DataStore::<Nil>::empty().push_back_from_iter::<order::Name1, _, _, _>(
vec!["First1", "Second1", "Third1"]
.iter()
.map(|&s| s.to_owned()),
);
let store = store.push_back_from_iter::<order::Name2, _, _, _>(
vec!["First2", "Second2", "Third2"]
.iter()
.map(|&s| s.to_owned()),
);
let store = store.push_back_from_iter::<order::Name3, _, _, _>(
vec!["First3", "Second3", "Third3"]
.iter()
.map(|&s| s.to_owned()),
);
let framed_data = Framed::<String, _>::new_melt(
Rc::new(Permutation::default()),
vec![
store.field::<order::Name1>(),
store.field::<order::Name2>(),
store.field::<order::Name3>(),
],
);
println!("{:?}", framed_data.to_vec());
assert_eq!(
framed_data.to_vec(),
vec![
"First1", "First2", "First3", "Second1", "Second2", "Second3", "Third1", "Third2",
"Third3",
]
);
}
#[test]
fn frame_view() {
let store = DataStore::<Nil>::empty().push_back_from_iter::<order::Name1, _, _, _>(
vec!["First1", "Second1", "Third1"]
.iter()
.map(|&s| s.to_owned()),
);
let dv = store.into_view();
println!("{}", dv.nrows());
let view_in_frame: DataFrame<_, _> = dv.into();
println!("{}", view_in_frame.nrows());
let final_view = view_in_frame.into_view();
println!("{}", final_view);
}
}