varpulis_runtime/lib.rs
1#![allow(missing_docs)]
2//! # Varpulis Runtime
3//!
4//! High-performance execution engine for VPL programs.
5//!
6//! This crate is the heart of Varpulis, providing:
7//!
8//! - **Stream Processing**: Real-time event processing with filtering and transformation
9//! - **SASE+ Pattern Matching**: Complex event detection with sequences, Kleene closures, and negation
10//! - **Windowed Aggregations**: Time and count-based windows with SIMD-optimized aggregations
11//! - **Connectors**: MQTT, HTTP, and file-based event sources/sinks
12//!
13//! ## Features
14//!
15//! | Feature | Description |
16//! |---------|-------------|
17//! | `mqtt` | MQTT connector support (rumqttc) |
18//! | `kafka` | Kafka connector support (rdkafka) |
19//! | `persistence` | RocksDB state persistence |
20//! | `database` | SQL database connectors (PostgreSQL, MySQL, SQLite) |
21//! | `redis` | Redis connector support |
22//! | `all-connectors` | Enable all connector features |
23//!
24//! ## Modules
25//!
26//! ### Core Processing
27//! - [`engine`]: Main execution engine, compiles and runs VPL programs
28//! - [`event`]: Event structure and field access
29//! - [`stream`]: Stream abstraction for event flows
30//!
31//! ### Pattern Matching
32//! - [`sase`]: SASE+ pattern matching (SEQ, AND, OR, NOT, Kleene+/*)
33//! - [`sequence`]: Sequence pattern tracking
34//!
35//! ### Windowing & Aggregation
36//! - [`window`]: Tumbling, sliding, and count-based windows
37//! - [`aggregation`]: Aggregation functions (sum, avg, min, max, stddev, percentile)
38//! - [`simd`]: SIMD-optimized operations using AVX2
39//!
40//! ### Advanced Features
41//! //! - [`join`]: Multi-stream join operations
42//!
43//! ### Multi-Query Trend Aggregation
44//! - [`greta`]: GRETA baseline aggregation (VLDB 2017)
45//! - [`hamlet`]: Hamlet shared aggregation with graphlets (SIGMOD 2021) - **recommended**
46//! - [`zdd_unified`]: ZDD-based aggregation (experimental, for research)
47//!
48//! ### I/O & Connectors
49//! - [`connector`]: Source and sink connectors (MQTT, HTTP, Kafka)
50//! - [`sink`]: Output sinks (console, file, HTTP webhook)
51//! - [`event_file`]: Event file parsing and streaming
52//!
53//! ### Infrastructure
54//! - [`worker_pool`]: Parallel processing with backpressure
55//! - [`persistence`]: State checkpointing (RocksDB, memory)
56//! - [`metrics`]: Prometheus metrics
57//! - [`timer`]: Timer management for timeouts
58//! - [`simulator`]: Event simulation for demos
59//!
60//! ## Quick Start
61//!
62//! ```rust,no_run
63//! use varpulis_runtime::{Engine, Event};
64//! use varpulis_parser::parse;
65//! use tokio::sync::mpsc;
66//!
67//! #[tokio::main]
68//! async fn main() {
69//! // Parse a VPL program
70//! let program = parse(r#"
71//! stream HighTemp = SensorReading
72//! .where(temperature > 100)
73//! .emit(sensor: sensor_id, temp: temperature)
74//! "#).unwrap();
75//!
76//! // Create engine with output channel
77//! let (output_tx, mut output_rx) = mpsc::channel(100);
78//! let mut engine = Engine::new(output_tx);
79//! engine.load(&program).unwrap();
80//!
81//! // Process an event
82//! let event = Event::new("SensorReading")
83//! .with_field("temperature", 105.5)
84//! .with_field("sensor_id", "S1");
85//! engine.process(event).await.unwrap();
86//!
87//! // Receive output event
88//! if let Some(output) = output_rx.recv().await {
89//! println!("Output: {} {:?}", output.event_type, output.data);
90//! }
91//! }
92//! ```
93//!
94//! ## Performance
95//!
96//! - SIMD-optimized aggregations (4x speedup with AVX2)
97//! - Incremental aggregation for sliding windows
98//! - Zero-copy event sharing via `Arc<Event>`
99//! - Parallel worker pools with backpressure
100//!
101//! ## See Also
102//!
103//! - [`varpulis_core`](../varpulis_core): Core types and AST
104//! - [`varpulis_parser`](../varpulis_parser): Parsing VPL
105//! - [`varpulis_cli`](../varpulis_cli): Command-line interface
106
107pub mod aggregation;
108pub mod backpressure;
109pub mod codec;
110pub mod columnar;
111pub mod context;
112pub use varpulis_dead_letter as dead_letter;
113pub mod engine;
114pub use varpulis_enrichment as enrichment;
115pub mod event;
116pub mod event_file;
117pub mod greta;
118pub use varpulis_hamlet as hamlet;
119pub mod health;
120pub mod join;
121pub mod metrics;
122pub mod persistence;
123pub use varpulis_pst as pst;
124pub use varpulis_sase as sase;
125pub mod sase_persistence;
126pub mod scoring;
127pub mod sequence;
128pub use varpulis_simd as simd;
129pub mod simulator;
130pub mod sink;
131pub mod stream;
132pub mod tenant;
133pub mod testing;
134pub mod timer;
135pub mod udf;
136pub mod vpl_test;
137pub mod watermark;
138pub mod window;
139pub mod worker_pool;
140pub mod zdd_unified;
141
142// Re-export from varpulis-connectors for backwards compatibility
143// Columnar storage for SIMD-optimized aggregations
144pub use columnar::{Column, ColumnarAccess, ColumnarBuffer, ColumnarCheckpoint};
145pub use context::{
146 CheckpointAck, CheckpointBarrier, CheckpointCoordinator, ContextConfig, ContextMap,
147 ContextMessage, ContextOrchestrator, ContextRuntime, DispatchError, EventTypeRouter,
148};
149pub use engine::error::EngineError;
150pub use engine::{Engine, EngineBuilder, ReloadReport, SourceBinding};
151pub use event::{Event, SharedEvent};
152pub use event_file::StreamingEventReader;
153pub use metrics::Metrics;
154// Persistence exports (always available, RocksDB impl requires "persistence" feature)
155#[cfg(feature = "persistence")]
156pub use persistence::RocksDbStore;
157pub use persistence::{
158 Checkpoint, CheckpointConfig, CheckpointManager, FileStore, MemoryStore, StateStore, StoreError,
159};
160pub use sink::{ConsoleSink, FileSink, HttpSink, MultiSink};
161pub use stream::Stream;
162// Multi-tenant SaaS support
163pub use tenant::{
164 hash_api_key, shared_tenant_manager, shared_tenant_manager_with_store, Pipeline,
165 PipelineSnapshot, PipelineStatus, SharedTenantManager, Tenant, TenantError, TenantId,
166 TenantManager, TenantQuota, TenantSnapshot, TenantUsage,
167};
168pub use timer::{spawn_timer, TimerManager};
169pub use varpulis_connectors as connector;
170pub use varpulis_connectors::{circuit_breaker, converter, limits, Sink, SinkError};
171pub use window::{
172 CountWindow, DelayBuffer, IncrementalAggregates, IncrementalSlidingWindow,
173 PartitionedDelayBuffer, PartitionedPreviousValueTracker, PartitionedSessionWindow,
174 PartitionedSlidingWindow, PartitionedTumblingWindow, PreviousValueTracker, SessionWindow,
175 SlidingCountWindow, SlidingWindow, TumblingWindow,
176};
177pub use worker_pool::{
178 BackpressureStrategy, PoolBackpressureError, WorkerPool, WorkerPoolConfig, WorkerPoolMetrics,
179 WorkerState, WorkerStatus,
180};