hyperi-rustlib 2.6.0

Opinionated Rust framework for high-throughput data pipelines at PB scale. Auto-wiring config, logging, metrics, tracing, health, and graceful shutdown — built from many years of production infrastructure experience.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

// Project:   hyperi-rustlib
// File:      src/worker/mod.rs
// Purpose:   Adaptive worker pool with hybrid rayon + tokio execution
// Language:  Rust
//
// License:   FSL-1.1-ALv2
// Copyright: (c) 2026 HYPERI PTY LIMITED

//! Adaptive worker pool and batch processing framework.
//!
//! Two layers:
//!
//! - **Generic:** [`AdaptiveWorkerPool`] provides CPU-saturating parallelism via
//!   rayon (CPU-bound) + tokio (async I/O), with reactive pressure-based scaling.
//!   Useful for any workload — not DFE-specific.
//!
//! - **Opinionated:** [`BatchProcessor`] trait + [`BatchPipeline`] provide a
//!   structured parallel-then-sequential pipeline for DFE apps. Apps implement
//!   `BatchProcessor` for their domain; the pipeline handles stats, scaling,
//!   and batch orchestration. [`PipelineStats`] provides common atomic counters.
//!
//! ## Quick Start
//!
//! ```rust,ignore
//! use hyperi_rustlib::worker::{AdaptiveWorkerPool, WorkerPoolConfig};
//!
//! let pool = AdaptiveWorkerPool::from_cascade("worker_pool")?;
//! pool.register_metrics(&metrics_manager);
//! pool.start_scaling_loop(shutdown_token.clone());
//!
//! // CPU-bound parallel transform (rayon)
//! let results = pool.process_batch(&messages, |msg| {
//!     parse_and_transform(msg)
//! });
//!
//! // Async parallel enrichment (tokio)
//! let enriched = pool.fan_out_async(&items, |item| async move {
//!     enrich(item).await
//! }).await;
//! ```

mod accumulator;
mod batch;
mod config;
pub mod engine;
pub(crate) mod metrics;
pub mod ndjson;
mod pool;
pub(crate) mod scaler;
mod stats;

pub use accumulator::{AccumulatorConfig, AccumulatorFull, BatchAccumulator, BatchDrainer};
pub use batch::{BatchPipeline, BatchProcessor};
pub use config::WorkerPoolConfig;
#[cfg(feature = "transport")]
pub use engine::EngineError;
pub use engine::{
    BatchEngine, BatchProcessingConfig, FieldInterner, MessageMetadata, ParsedMessage,
    PreRouteFilterConfig, RawMessage,
};
pub use pool::AdaptiveWorkerPool;
pub use scaler::{ScalingDecision, ScalingInput};
pub use stats::{PipelineStats, PipelineStatsSnapshot};