Skip to main content

oxigeo_etl/
lib.rs

1//! OxiGeo ETL - Streaming ETL framework for geospatial data processing
2//!
3//! This crate provides a comprehensive ETL (Extract, Transform, Load) framework
4//! for continuous geospatial data processing with OxiGeo.
5//!
6//! # Features
7//!
8//! - **Async Streaming**: Built on tokio for high-performance async I/O
9//! - **Backpressure Handling**: Automatic backpressure management
10//! - **Error Recovery**: Configurable error handling and retry logic
11//! - **Checkpointing**: State persistence for fault tolerance
12//! - **Monitoring**: Built-in metrics and logging
13//! - **Resource Limits**: Control parallelism and memory usage
14//!
15//! # Architecture
16//!
17//! The ETL framework consists of several key components:
18//!
19//! - **Sources**: Data inputs (files, HTTP, S3, STAC, PostGIS)
20//! - **Transforms**: Data transformations (map, filter, window, join, etc.)
21//! - **Sinks**: Data outputs (files, S3, PostGIS)
22//! - **Pipeline**: Fluent API for composing ETL workflows
23//! - **Stream**: Async stream processing with backpressure
24//! - **Scheduler**: Task scheduling and execution
25//!
26//! # Example
27//!
28//! ```rust,no_run
29//! use oxigeo_etl::*;
30//! use oxigeo_etl::source::FileSource;
31//! use oxigeo_etl::sink::FileSink;
32//! use oxigeo_etl::error::TransformError;
33//! use std::path::PathBuf;
34//!
35//! # async fn example() -> Result<()> {
36//! // Build ETL pipeline
37//! let pipeline = Pipeline::builder()
38//!     .source(Box::new(FileSource::new(PathBuf::from("input.json"))))
39//!     .map("uppercase".to_string(), |item| {
40//!         Box::pin(async move {
41//!             let s = String::from_utf8(item).map_err(|e| {
42//!                 TransformError::InvalidInput {
43//!                     message: e.to_string(),
44//!                 }
45//!             })?;
46//!             Ok(s.to_uppercase().into_bytes())
47//!         })
48//!     })
49//!     .filter("non_empty".to_string(), |item| {
50//!         let is_empty = item.is_empty();
51//!         Box::pin(async move { Ok(!is_empty) })
52//!     })
53//!     .sink(Box::new(FileSink::new(PathBuf::from("output.json"))))
54//!     .with_checkpointing()
55//!     .buffer_size(1000)
56//!     .build()?;
57//!
58//! // Execute pipeline
59//! let stats = pipeline.run().await?;
60//! println!("Processed {} items", stats.items_processed());
61//! # Ok(())
62//! # }
63//! ```
64//!
65//! # Feature Flags
66//!
67//! - `std` (default): Enable standard library support
68//! - `postgres`: Enable PostgreSQL/PostGIS support
69//! - `s3`: Enable Amazon S3 support
70//! - `stac`: Enable STAC catalog support
71//! - `http`: Enable HTTP source support
72//! - `scheduler`: Enable cron-based scheduling
73//! - `all`: Enable all optional features
74//!
75//! The `kafka` feature (Kafka source and sink) was **removed in 0.2.1** together
76//! with the retired `oxigeo-kafka` crate; see the workspace CHANGELOG.
77
78#![warn(missing_docs)]
79#![cfg_attr(not(feature = "std"), no_std)]
80
81// Re-export key types
82pub use error::{EtlError, Result};
83pub use pipeline::{ExecutionMode, Pipeline, PipelineBuilder, PipelineConfig, PipelineStats};
84pub use scheduler::{Schedule, Scheduler, TaskConfig, TaskResult};
85pub use sink::Sink;
86pub use source::Source;
87pub use stream::{
88    BoxStream, BufferedStream, ParallelProcessor, StateManager, StreamConfig, StreamItem,
89    StreamProcessor,
90};
91pub use transform::Transform;
92
93// Modules
94pub mod error;
95pub mod operators;
96pub mod pipeline;
97pub mod scheduler;
98pub mod sink;
99pub mod source;
100pub mod stream;
101pub mod transform;
102
103/// Prelude module for convenient imports
104pub mod prelude {
105    pub use crate::error::{EtlError, Result};
106    pub use crate::operators::{aggregate::*, filter::*, join::*, map::*, window::*};
107    pub use crate::pipeline::{ExecutionMode, Pipeline, PipelineBuilder};
108    pub use crate::scheduler::{Schedule, Scheduler, TaskConfig};
109    pub use crate::sink::{FileSink, Sink};
110    pub use crate::source::{FileSource, Source};
111    pub use crate::stream::{StreamConfig, StreamItem, StreamProcessor};
112    pub use crate::transform::Transform;
113
114    #[cfg(feature = "postgres")]
115    pub use crate::sink::PostGisSink;
116
117    #[cfg(feature = "s3")]
118    pub use crate::sink::S3Sink;
119
120    #[cfg(feature = "stac")]
121    pub use crate::source::StacSource;
122
123    #[cfg(feature = "http")]
124    pub use crate::source::HttpSource;
125}
126
127/// Version information
128pub const VERSION: &str = env!("CARGO_PKG_VERSION");
129
130/// Crate name
131pub const CRATE_NAME: &str = env!("CARGO_PKG_NAME");
132
133#[cfg(test)]
134mod tests {
135    use super::*;
136
137    #[test]
138    fn test_version() {
139        assert!(!VERSION.is_empty());
140    }
141
142    #[test]
143    fn test_crate_name() {
144        assert_eq!(CRATE_NAME, "oxigeo-etl");
145    }
146}