tenshift-core 0.1.2

//! Pipeline  -  the composable data loading engine.
//!
//! A pipeline is a chain: `Source -> [Workers] -> [Collector] -> Consumer`.
//! Multiple worker threads load data and apply stateless transforms in parallel.
//! A single collector thread applies ordering-sensitive stages such as shuffle,
//! batching, and collation.
//!
//! # Threading Model
//!
//! ```text
//! ┌──────────────┐     ┌──────────────┐     ┌──────────────┐     ┌──────────┐
//! │ Source Thread│────▶│Worker Threads│────▶│Collector     │────▶│ Consumer │
//! │  (1 thread)  │     │ (N parallel) │     │Thread (1)    │     │ (caller) │
//! └──────────────┘     └──────────────┘     └──────────────┘     └──────────┘
//!       │                     │                    │
//!       ▼                     ▼                    ▼
//!   File I/O,              CPU-bound           Ordering-sensitive
//!   network               transforms          (shuffle, batch)
//! ```
//!
//! # Configuration
//!
//! Use [`Pipeline`] builder methods to customize:
//!
//! ```rust
//! use tenshift_core::Pipeline;
//! use tenshift_core::sources::MemorySource;
//! use tenshift_core::sample::Sample;
//!
//! let pipeline = Pipeline::from_source(MemorySource::new("demo", Vec::<Sample>::new()))
//!     .workers(4)           // Number of parallel worker threads
//!     .prefetch(8)          // Items to prefetch (see auto-scaling docs)
//!     .batch(32)            // Batch size for collation
//!     .shuffle(1000);       // Shuffle buffer size
//! ```
//!
//! See the [crate-level documentation](crate) for detailed auto-scaling defaults.

mod builder;
mod collate;
mod collector;
mod config;
mod core_types;
mod flow_control;
mod iterator;
mod reorder_buffer;
mod source_thread;
mod utils;
mod worker;

pub use builder::Pipeline;
pub use core_types::CollateMode;
pub(crate) use collate::*;
pub use config::{ErrorPolicy, PipelineConfig, DEFAULT_SHUFFLE_SEED, MAX_LOAD_FILE_SIZE};
pub(crate) use core_types::*;
pub(crate) use flow_control::*;
pub use iterator::{NextTimeoutError, PipelineIterator, PipelineStats};
pub(crate) use reorder_buffer::*;
pub(crate) use utils::*;
mod executor;