varpulis_runtime/lib.rs
1#![allow(missing_docs)]
2//! # Varpulis Runtime
3//!
4//! High-performance execution engine for VPL programs.
5//!
6//! This crate is the heart of Varpulis, providing:
7//!
8//! - **Stream Processing**: Real-time event processing with filtering and transformation
9//! - **SASE+ Pattern Matching**: Complex event detection with sequences, Kleene closures, and negation
10//! - **Windowed Aggregations**: Time and count-based windows with SIMD-optimized aggregations
11//! - **Connectors**: MQTT, HTTP, and file-based event sources/sinks
12//!
13//! ## Features
14//!
15//! | Feature | Description |
16//! |---------|-------------|
17//! | `mqtt` | MQTT connector support (rumqttc) |
18//! | `kafka` | Kafka connector support (rdkafka) |
19//! | `persistence` | RocksDB state persistence |
20//! | `database` | SQL database connectors (PostgreSQL, MySQL, SQLite) |
21//! | `redis` | Redis connector support |
22//! | `all-connectors` | Enable all connector features |
23//!
24//! ## Modules
25//!
26//! ### Core Processing
27//! - [`engine`]: Main execution engine, compiles and runs VPL programs
28//! - [`event`]: Event structure and field access
29//! - [`stream`]: Stream abstraction for event flows
30//!
31//! ### Pattern Matching
32//! - [`sase`]: SASE+ pattern matching (SEQ, AND, OR, NOT, Kleene+/*)
33//! - [`sequence`]: Sequence pattern tracking
34//!
35//! ### Windowing & Aggregation
36//! - [`window`]: Tumbling, sliding, and count-based windows
37//! - [`aggregation`]: Aggregation functions (sum, avg, min, max, stddev, percentile)
38//! - [`simd`]: SIMD-optimized operations using AVX2
39//!
40//! ### Advanced Features
41//! //! - [`join`]: Multi-stream join operations
42//!
43//! ### Multi-Query Trend Aggregation
44//! - [`greta`]: GRETA baseline aggregation (VLDB 2017)
45//! - [`hamlet`]: Hamlet shared aggregation with graphlets (SIGMOD 2021) - **recommended**
46//! - [`zdd_unified`]: ZDD-based aggregation (experimental, for research)
47//!
48//! ### I/O & Connectors
49//! - [`connector`]: Source and sink connectors (MQTT, HTTP, Kafka)
50//! - [`sink`]: Output sinks (console, file, HTTP webhook)
51//! - [`event_file`]: Event file parsing and streaming
52//!
53//! ### Infrastructure
54//! - [`worker_pool`]: Parallel processing with backpressure
55//! - [`persistence`]: State checkpointing (RocksDB, memory)
56//! - [`metrics`]: Prometheus metrics
57//! - [`timer`]: Timer management for timeouts
58//! - [`simulator`]: Event simulation for demos
59//!
60//! ## Quick Start
61//!
62//! ```rust,no_run
63//! use varpulis_runtime::{Engine, Event};
64//! use varpulis_parser::parse;
65//! use tokio::sync::mpsc;
66//!
67//! #[tokio::main]
68//! async fn main() {
69//! // Parse a VPL program
70//! let program = parse(r#"
71//! stream HighTemp = SensorReading
72//! .where(temperature > 100)
73//! .emit(sensor: sensor_id, temp: temperature)
74//! "#).unwrap();
75//!
76//! // Create engine with output channel
77//! let (output_tx, mut output_rx) = mpsc::channel(100);
78//! let mut engine = Engine::new(output_tx);
79//! engine.load(&program).unwrap();
80//!
81//! // Process an event
82//! let event = Event::new("SensorReading")
83//! .with_field("temperature", 105.5)
84//! .with_field("sensor_id", "S1");
85//! engine.process(event).await.unwrap();
86//!
87//! // Receive output event
88//! if let Some(output) = output_rx.recv().await {
89//! println!("Output: {} {:?}", output.event_type, output.data);
90//! }
91//! }
92//! ```
93//!
94//! ## Performance
95//!
96//! - SIMD-optimized aggregations (4x speedup with AVX2)
97//! - Incremental aggregation for sliding windows
98//! - Zero-copy event sharing via `Arc<Event>`
99//! - Parallel worker pools with backpressure
100//!
101//! ## See Also
102//!
103//! - [`varpulis_core`](../varpulis_core): Core types and AST
104//! - [`varpulis_parser`](../varpulis_parser): Parsing VPL
105//! - [`varpulis_cli`](../varpulis_cli): Command-line interface
106
107// ---- Core modules (always available, no async runtime needed) ----
108pub mod aggregation;
109pub mod codec;
110pub mod columnar;
111pub mod limits;
112pub use varpulis_dead_letter as dead_letter;
113pub mod engine;
114pub mod event;
115pub mod event_file;
116pub mod greta;
117pub use varpulis_hamlet as hamlet;
118pub mod health;
119pub mod join;
120pub mod persistence;
121pub use varpulis_pst as pst;
122pub use varpulis_sase as sase;
123pub mod sase_persistence;
124pub mod scoring;
125pub mod sequence;
126pub use varpulis_simd as simd;
127pub mod udf;
128pub mod vpl_test;
129pub mod watermark;
130pub mod window;
131pub mod zdd_unified;
132
133// ---- Async-runtime modules (require tokio, not available in WASM) ----
134#[cfg(feature = "async-runtime")]
135pub mod backpressure;
136#[cfg(feature = "async-runtime")]
137pub mod context;
138#[cfg(feature = "async-runtime")]
139pub use varpulis_enrichment as enrichment;
140#[cfg(feature = "async-runtime")]
141pub mod interactive;
142#[cfg(feature = "async-runtime")]
143pub mod metrics;
144#[cfg(feature = "async-runtime")]
145pub mod simulator;
146#[cfg(feature = "async-runtime")]
147pub mod sink;
148#[cfg(feature = "async-runtime")]
149pub mod stream;
150#[cfg(feature = "async-runtime")]
151pub mod tenant;
152#[cfg(feature = "async-runtime")]
153pub mod testing;
154#[cfg(feature = "async-runtime")]
155pub mod timer;
156#[cfg(feature = "async-runtime")]
157pub mod worker_pool;
158
159// ---- Core re-exports (always available) ----
160pub use columnar::{Column, ColumnarAccess, ColumnarBuffer, ColumnarCheckpoint};
161// ---- Async-runtime re-exports (require tokio) ----
162#[cfg(feature = "async-runtime")]
163pub use context::{
164 CheckpointAck, CheckpointBarrier, CheckpointCoordinator, ContextConfig, ContextMap,
165 ContextMessage, ContextOrchestrator, ContextRuntime, DispatchError, EventTypeRouter,
166};
167pub use engine::error::EngineError;
168#[cfg(feature = "async-runtime")]
169pub use engine::EngineBuilder;
170pub use engine::{Engine, ReloadReport, SourceBinding};
171pub use event::{Event, SharedEvent};
172pub use event_file::StreamingEventReader;
173#[cfg(feature = "async-runtime")]
174pub use metrics::Metrics;
175// Persistence exports (always available, RocksDB impl requires "persistence" feature)
176#[cfg(feature = "persistence")]
177pub use persistence::RocksDbStore;
178pub use persistence::{
179 Checkpoint, CheckpointConfig, CheckpointManager, FileStore, MemoryStore, StateStore, StoreError,
180};
181#[cfg(feature = "async-runtime")]
182pub use sink::{ConsoleSink, FileSink, HttpSink, MultiSink};
183#[cfg(feature = "async-runtime")]
184pub use stream::Stream;
185#[cfg(feature = "async-runtime")]
186pub use tenant::{
187 hash_api_key, shared_tenant_manager, shared_tenant_manager_with_store, Pipeline,
188 PipelineSnapshot, PipelineStatus, SharedTenantManager, Tenant, TenantError, TenantId,
189 TenantManager, TenantQuota, TenantSnapshot, TenantUsage,
190};
191#[cfg(feature = "async-runtime")]
192pub use timer::{spawn_timer, TimerManager};
193#[cfg(feature = "async-runtime")]
194pub use varpulis_connectors as connector;
195#[cfg(feature = "async-runtime")]
196pub use varpulis_connectors::{circuit_breaker, converter, Sink, SinkError};
197pub use window::{
198 CountWindow, DelayBuffer, IncrementalAggregates, IncrementalSlidingWindow,
199 PartitionedDelayBuffer, PartitionedPreviousValueTracker, PartitionedSessionWindow,
200 PartitionedSlidingWindow, PartitionedTumblingWindow, PreviousValueTracker, SessionWindow,
201 SlidingCountWindow, SlidingWindow, TumblingWindow,
202};
203#[cfg(feature = "async-runtime")]
204pub use worker_pool::{
205 BackpressureStrategy, PoolBackpressureError, WorkerPool, WorkerPoolConfig, WorkerPoolMetrics,
206 WorkerState, WorkerStatus,
207};