csvs_convert/
lib.rs

1//! Converts CSV files into XLSX/SQLITE/POSTGRESQL/PARQUET fast.
2//!
3//! ## Aims
4
5//! * Thorough type guessing of CSV columns, so there is no need to configure types of each field. Scans whole file first to make sure all types in a column are consistent. Can detect over 30 date/time formats as well as JSON data.
6//! * Quick conversions/type guessing (uses rust underneath). Uses fast methods specific for each output format:
7//!     * `copy` for postgres
8//!     * Prepared statements for sqlite using c API.
9//!     * Arrow reader for parquet
10//!     * Write only mode for libxlsxwriter
11//! * Tries to limit errors when inserting data into database by resorting to "text" if type guessing can't determine a more specific type.
12//! * When inserting into existing databases automatically migrate schema of target to allow for new data (`evolve` option).
13//! * Memory efficient. All csvs and outputs are streamed so all conversions should take up very little memory.
14//! * Gather stats and information about CSV files into datapacakge.json file which can use it for customizing conversion.
15//!
16//! ## Drawbacks
17//!
18//! * CSV files currently need header rows.
19//! * Whole file needs to be on disk as whole CSV is analyzed therefore files are read twice.
20
21#[cfg(not(target_family = "wasm"))]
22#[cfg(feature = "converters")]
23mod converters;
24
25mod describe;
26mod describe_csv;
27mod describer;
28
29#[cfg(not(target_family = "wasm"))]
30#[cfg(feature = "converters")]
31mod zip_dir;
32
33pub use describe::{
34    describe_files, make_datapackage, output_datapackage, DescribeError, Options as DescribeOptions,
35};
36pub use describer::{Describer, Options as DescriberOptions};
37
38#[cfg(feature = "converters")]
39#[cfg(not(target_family = "wasm"))]
40pub use converters::{
41    csvs_to_postgres, csvs_to_postgres_with_options,
42    csvs_to_sqlite, csvs_to_sqlite_with_options, csvs_to_xlsx, csvs_to_xlsx_with_options,
43    csvs_to_ods, csvs_to_ods_with_options,
44    datapackage_to_postgres,
45    datapackage_to_postgres_with_options, datapackage_to_sqlite,
46    datapackage_to_sqlite_with_options, datapackage_to_xlsx, datapackage_to_xlsx_with_options,
47    datapackage_to_ods, datapackage_to_ods_with_options,
48    merge_datapackage, merge_datapackage_jsons, merge_datapackage_with_options,
49    Error, Options
50};
51
52#[cfg(feature = "parquet")]
53#[cfg(not(target_family = "wasm"))]
54pub use converters::{
55    csvs_to_parquet, csvs_to_parquet_with_options, 
56    datapackage_to_parquet, datapackage_to_parquet_with_options};
57
csvs_convert/lib.rs

csvs_convert/
lib.rs
