exarrow_rs/import/
mod.rs

1//! Import functionality for Exasol data.
2//!
3//! This module provides utilities for importing data into Exasol from various sources
4//! including CSV files, Parquet files, streams, iterators, and custom callbacks.
5//!
6//! # Overview
7//!
8//! The import module supports:
9//! - CSV import from file paths
10//! - CSV import from async readers/streams
11//! - CSV import from iterators
12//! - CSV import with custom callbacks for data generation
13//! - Parquet import (converted to CSV on-the-fly)
14//! - Compression support (gzip, bzip2)
15//!
16//! # Architecture
17//!
18//! Exasol's IMPORT command only accepts CSV format over HTTP. For non-CSV sources
19//! (like Parquet), data is converted to CSV on-the-fly during streaming.
20//!
21//! The import process works as follows:
22//! 1. Start an HTTP transport server on a local port
23//! 2. Execute an IMPORT SQL statement via WebSocket that points to our server
24//! 3. Stream data through the HTTP transport to Exasol
25//!
26//! # Example
27//!
28//! ```no_run
29//! use exarrow_rs::import::{ParquetImportOptions, import_from_parquet};
30//! use std::path::Path;
31//!
32//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
33//! // Import from a Parquet file
34//! // let rows = import_from_parquet(&mut session, "my_table", Path::new("data.parquet"), ParquetImportOptions::default()).await?;
35//! // println!("Imported {} rows", rows);
36//! # Ok(())
37//! # }
38//! ```
39
40pub mod arrow;
41pub mod csv;
42pub mod parallel;
43pub mod parquet;
44pub mod source;
45
46pub use arrow::{
47    import_from_arrow_ipc, import_from_record_batch, import_from_record_batches,
48    ArrowImportOptions, ArrowToCsvWriter, CsvWriterOptions,
49};
50
51pub use csv::{
52    import_from_callback, import_from_file, import_from_files, import_from_iter,
53    import_from_stream, CsvImportOptions, DataPipeSender,
54};
55
56pub use parquet::{
57    import_from_parquet, import_from_parquet_files, import_from_parquet_stream,
58    ParquetImportOptions,
59};
60
61// Re-export ColumnNameMode for convenient access
62pub use crate::types::ColumnNameMode;
63
64pub use parallel::{ImportFileEntry, ParallelTransportPool};
65pub use source::IntoFileSources;
66
67use thiserror::Error;
68
69/// Errors that can occur during import operations.
70#[derive(Error, Debug)]
71pub enum ImportError {
72    /// IO error during file operations
73    #[error("IO error: {0}")]
74    IoError(#[from] std::io::Error),
75
76    /// Parquet file reading error
77    #[error("Parquet error: {0}")]
78    ParquetError(String),
79
80    /// Arrow conversion error
81    #[error("Arrow error: {0}")]
82    ArrowError(String),
83
84    /// Transport error during HTTP streaming
85    #[error("Transport error: {0}")]
86    TransportError(#[from] crate::error::TransportError),
87
88    /// Query execution error
89    #[error("Query error: {0}")]
90    QueryError(String),
91
92    /// Data conversion error
93    #[error("Conversion error: {0}")]
94    ConversionError(String),
95
96    /// Invalid configuration
97    #[error("Invalid configuration: {0}")]
98    InvalidConfig(String),
99
100    /// CSV writing error
101    #[error("CSV write error: {0}")]
102    CsvWriteError(String),
103
104    /// Arrow IPC reading error
105    #[error("Arrow IPC error: {0}")]
106    ArrowIpcError(String),
107
108    /// SQL execution error
109    #[error("SQL execution failed: {0}")]
110    SqlError(String),
111
112    /// HTTP transport server failed
113    #[error("HTTP transport failed: {0}")]
114    HttpTransportError(String),
115
116    /// Data streaming error
117    #[error("Data streaming error: {0}")]
118    StreamError(String),
119
120    /// Compression error
121    #[error("Compression error: {0}")]
122    CompressionError(String),
123
124    /// Invalid session state
125    #[error("Invalid session state: {0}")]
126    InvalidSessionState(String),
127
128    /// Channel communication error
129    #[error("Channel error: {0}")]
130    ChannelError(String),
131
132    /// Parallel import error (connection, streaming, or conversion failure)
133    #[error("Parallel import error: {0}")]
134    ParallelImportError(String),
135
136    /// Schema inference failed (could not read metadata or convert types)
137    #[error("Schema inference failed: {0}")]
138    SchemaInferenceError(String),
139
140    /// Schema mismatch between multiple files
141    #[error("Schema mismatch between files: {0}")]
142    SchemaMismatchError(String),
143}
144
145impl From<::arrow::error::ArrowError> for ImportError {
146    fn from(err: ::arrow::error::ArrowError) -> Self {
147        ImportError::ArrowError(err.to_string())
148    }
149}
150
151impl From<::parquet::errors::ParquetError> for ImportError {
152    fn from(err: ::parquet::errors::ParquetError) -> Self {
153        ImportError::ParquetError(err.to_string())
154    }
155}
exarrow_rs/import/mod.rs

exarrow_rs/import/
mod.rs
