oxigeo_etl/lib.rs
1//! OxiGeo ETL - Streaming ETL framework for geospatial data processing
2//!
3//! This crate provides a comprehensive ETL (Extract, Transform, Load) framework
4//! for continuous geospatial data processing with OxiGeo.
5//!
6//! # Features
7//!
8//! - **Async Streaming**: Built on tokio for high-performance async I/O
9//! - **Backpressure Handling**: Automatic backpressure management
10//! - **Error Recovery**: Configurable error handling and retry logic
11//! - **Checkpointing**: State persistence for fault tolerance
12//! - **Monitoring**: Built-in metrics and logging
13//! - **Resource Limits**: Control parallelism and memory usage
14//!
15//! # Architecture
16//!
17//! The ETL framework consists of several key components:
18//!
19//! - **Sources**: Data inputs (files, HTTP, S3, STAC, PostGIS)
20//! - **Transforms**: Data transformations (map, filter, window, join, etc.)
21//! - **Sinks**: Data outputs (files, S3, PostGIS)
22//! - **Pipeline**: Fluent API for composing ETL workflows
23//! - **Stream**: Async stream processing with backpressure
24//! - **Scheduler**: Task scheduling and execution
25//!
26//! # Example
27//!
28//! ```rust,no_run
29//! use oxigeo_etl::*;
30//! use oxigeo_etl::source::FileSource;
31//! use oxigeo_etl::sink::FileSink;
32//! use oxigeo_etl::error::TransformError;
33//! use std::path::PathBuf;
34//!
35//! # async fn example() -> Result<()> {
36//! // Build ETL pipeline
37//! let pipeline = Pipeline::builder()
38//! .source(Box::new(FileSource::new(PathBuf::from("input.json"))))
39//! .map("uppercase".to_string(), |item| {
40//! Box::pin(async move {
41//! let s = String::from_utf8(item).map_err(|e| {
42//! TransformError::InvalidInput {
43//! message: e.to_string(),
44//! }
45//! })?;
46//! Ok(s.to_uppercase().into_bytes())
47//! })
48//! })
49//! .filter("non_empty".to_string(), |item| {
50//! let is_empty = item.is_empty();
51//! Box::pin(async move { Ok(!is_empty) })
52//! })
53//! .sink(Box::new(FileSink::new(PathBuf::from("output.json"))))
54//! .with_checkpointing()
55//! .buffer_size(1000)
56//! .build()?;
57//!
58//! // Execute pipeline
59//! let stats = pipeline.run().await?;
60//! println!("Processed {} items", stats.items_processed());
61//! # Ok(())
62//! # }
63//! ```
64//!
65//! # Feature Flags
66//!
67//! - `std` (default): Enable standard library support
68//! - `postgres`: Enable PostgreSQL/PostGIS support
69//! - `s3`: Enable Amazon S3 support
70//! - `stac`: Enable STAC catalog support
71//! - `http`: Enable HTTP source support
72//! - `scheduler`: Enable cron-based scheduling
73//! - `all`: Enable all optional features
74//!
75//! The `kafka` feature (Kafka source and sink) was **removed in 0.2.1** together
76//! with the retired `oxigeo-kafka` crate; see the workspace CHANGELOG.
77
78#![warn(missing_docs)]
79#![cfg_attr(not(feature = "std"), no_std)]
80
81// Re-export key types
82pub use error::{EtlError, Result};
83pub use pipeline::{ExecutionMode, Pipeline, PipelineBuilder, PipelineConfig, PipelineStats};
84pub use scheduler::{Schedule, Scheduler, TaskConfig, TaskResult};
85pub use sink::Sink;
86pub use source::Source;
87pub use stream::{
88 BoxStream, BufferedStream, ParallelProcessor, StateManager, StreamConfig, StreamItem,
89 StreamProcessor,
90};
91pub use transform::Transform;
92
93// Modules
94pub mod error;
95pub mod operators;
96pub mod pipeline;
97pub mod scheduler;
98pub mod sink;
99pub mod source;
100pub mod stream;
101pub mod transform;
102
103/// Prelude module for convenient imports
104pub mod prelude {
105 pub use crate::error::{EtlError, Result};
106 pub use crate::operators::{aggregate::*, filter::*, join::*, map::*, window::*};
107 pub use crate::pipeline::{ExecutionMode, Pipeline, PipelineBuilder};
108 pub use crate::scheduler::{Schedule, Scheduler, TaskConfig};
109 pub use crate::sink::{FileSink, Sink};
110 pub use crate::source::{FileSource, Source};
111 pub use crate::stream::{StreamConfig, StreamItem, StreamProcessor};
112 pub use crate::transform::Transform;
113
114 #[cfg(feature = "postgres")]
115 pub use crate::sink::PostGisSink;
116
117 #[cfg(feature = "s3")]
118 pub use crate::sink::S3Sink;
119
120 #[cfg(feature = "stac")]
121 pub use crate::source::StacSource;
122
123 #[cfg(feature = "http")]
124 pub use crate::source::HttpSource;
125}
126
127/// Version information
128pub const VERSION: &str = env!("CARGO_PKG_VERSION");
129
130/// Crate name
131pub const CRATE_NAME: &str = env!("CARGO_PKG_NAME");
132
133#[cfg(test)]
134mod tests {
135 use super::*;
136
137 #[test]
138 fn test_version() {
139 assert!(!VERSION.is_empty());
140 }
141
142 #[test]
143 fn test_crate_name() {
144 assert_eq!(CRATE_NAME, "oxigeo-etl");
145 }
146}