1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
#![recursion_limit = "128"]
extern crate proc_macro;
extern crate proc_macro2;
extern crate syn;
#[macro_use]
extern crate quote;
extern crate parquet;
use syn::{parse_macro_input, Data, DataStruct, DeriveInput};
mod parquet_field;
#[proc_macro_derive(ParquetRecordWriter)]
pub fn parquet_record_writer(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
let input: DeriveInput = parse_macro_input!(input as DeriveInput);
let fields = match input.data {
Data::Struct(DataStruct { fields, .. }) => fields,
Data::Enum(_) => unimplemented!("Enum currently is not supported"),
Data::Union(_) => unimplemented!("Union currently is not supported"),
};
let field_infos: Vec<_> = fields
.iter()
.map(|f: &syn::Field| parquet_field::Field::from(f))
.collect();
let writer_snippets: Vec<proc_macro2::TokenStream> =
field_infos.iter().map(|x| x.writer_snippet()).collect();
let derived_for = input.ident;
let generics = input.generics;
let field_types: Vec<proc_macro2::TokenStream> =
field_infos.iter().map(|x| x.parquet_type()).collect();
(quote! {
impl#generics RecordWriter<#derived_for#generics> for &[#derived_for#generics] {
fn write_to_row_group(
&self,
row_group_writer: &mut Box<parquet::file::writer::RowGroupWriter>
) -> Result<(), parquet::errors::ParquetError> {
let mut row_group_writer = row_group_writer;
let records = &self;
#(
{
let mut some_column_writer = row_group_writer.next_column().unwrap();
if let Some(mut column_writer) = some_column_writer {
#writer_snippets
row_group_writer.close_column(column_writer)?;
} else {
return Err(parquet::errors::ParquetError::General("Failed to get next column".into()))
}
}
);*
Ok(())
}
fn schema(&self) -> Result<parquet::schema::types::TypePtr, parquet::errors::ParquetError> {
use parquet::schema::types::Type as ParquetType;
use parquet::schema::types::TypePtr;
use parquet::basic::LogicalType;
use parquet::basic::*;
let mut fields: Vec<TypePtr> = Vec::new();
#(
#field_types
);*;
let group = parquet::schema::types::Type::group_type_builder("rust_schema")
.with_fields(&mut fields)
.build()?;
Ok(group.into())
}
}
}).into()
}