hyperscan_tokio/

lib.rs

//! # hyperscan-tokio
//!
//! The most complete high-performance async regular expression matching library for Rust,
//! featuring both Hyperscan/VectorScan and Chimera (PCRE with capture groups) support.
//!
//! ## Core Features
//!
//! - **Dual Engine Support**: Both Hyperscan and Chimera in one library
//! - **PCRE Pattern Support**: Full PCRE syntax with capture groups via Chimera
//! - **Async-first design**: Built for Tokio with async/await support
//! - **High performance**: Leverages Hyperscan/VectorScan's SIMD acceleration
//! - **Multiple scanning modes**: Block, streaming, and vectored scanning
//! - **Thread-safe**: Databases can be shared across threads
//! - **Memory efficient**: Scratch space pooling and zero-copy operations
//!
//! ## Why hyperscan-tokio?
//!
//! This is the only Rust library that provides both:
//! 1. **Hyperscan**: For blazing-fast multi-pattern matching
//! 2. **Chimera**: For PCRE-compatible patterns with capture group support
//!
//! Whether you need simple pattern matching or complex PCRE expressions with
//! capture groups, this library has you covered.
//!
//! ## Optional Features
//!
//! - `chimera`: Enable PCRE-compatible patterns with capture groups
//! - `jemalloc`/`mimalloc`: Alternative memory allocators
//! - `arrow`: Apache Arrow integration for batch processing
//! - `simd-accel`: SIMD acceleration features
//!
//! ## Quick Start
//!
//! ### Hyperscan - Fast Multi-Pattern Matching
//!
//! ```rust,no_run
//! use hyperscan_tokio::prelude::*;
//!
//! # async fn example() -> Result<()> {
//! // Build a database with multiple patterns
//! let db = DatabaseBuilder::new()
//!     .add(Pattern::new(r"\d+").id(1).build()?)
//!     .add(Pattern::new(r"[a-z]+").id(2).build()?)
//!     .build()?;
//!
//! // Create a scanner
//! let scanner = Scanner::new(db)?;
//!
//! // Scan data - finds all patterns simultaneously
//! let matches = scanner.scan("abc 123 def 456").await?;
//! # Ok(())
//! # }
//! ```
//!
//! ### Chimera - PCRE Patterns with Capture Groups
//! 
//! To use Chimera, enable the `chimera` feature in your `Cargo.toml`:
//! 
//! ```toml
//! [dependencies]
//! hyperscan-tokio = { version = "0.1", features = ["chimera"] }
//! ```
//!
//! ```rust,no_run
//! #[cfg(feature = "chimera")]
//! use hyperscan_tokio::prelude::*;
//!
//! #[cfg(feature = "chimera")]
//! # fn example() -> Result<()> {
//! // Compile a PCRE pattern with named capture groups
//! let chimera = Chimera::compile(
//!     r"(?P<user>\w+)@(?P<domain>\w+\.\w+)",
//!     Flags::empty(),
//!     Mode::BLOCK
//! )?;
//!
//! // Scan and extract capture groups
//! chimera.scan(b"Contact: alice@example.com", |m| {
//!     if let Some(user) = m.group_by_name("user") {
//!         println!("User: {:?}", user.as_str(m.data));
//!     }
//!     if let Some(domain) = m.group_by_name("domain") {
//!         println!("Domain: {:?}", domain.as_str(m.data));
//!     }
//!     MatchControl::Continue
//! })?;
//! # Ok(())
//! # }
//! # #[cfg(not(feature = "chimera"))]
//! # fn example() {}
//! ```
//!
//! ## Performance
//!
//! This library achieves scanning speeds of 20+ GB/s on modern hardware:
//!
//! ```text
//! scanning_throughput/1048576     time:   [45.2 µs 45.8 µs 46.4 µs]
//!                                 thrpt:  [21.5 GiB/s 21.9 GiB/s 22.1 GiB/s]
//! ```
//!
//! ## Working with Patterns
//!
//! ### Pattern Building
//!
//! ```rust,no_run
//! use hyperscan_tokio::{Pattern, Flags};
//!
//! // Simple pattern
//! let p1 = Pattern::new(r"\d+").id(1).build()?;
//!
//! // Pattern with flags
//! let p2 = Pattern::new("test")
//!     .id(2)
//!     .flags(Flags::CASELESS | Flags::MULTILINE)
//!     .build()?;
//!
//! // Extended pattern with constraints
//! let p3 = Pattern::new(r"secret")
//!     .id(3)
//!     .min_offset(100)  // Must start after byte 100
//!     .max_offset(500)  // Must start before byte 500
//!     .min_length(10)   // Match must be at least 10 bytes
//!     .build()?;
//!
//! // Pattern with edit distance
//! let p4 = Pattern::new("password")
//!     .id(4)
//!     .edit_distance(2)  // Allow up to 2 character edits
//!     .build()?;
//! # Ok::<(), hyperscan_tokio::Error>(())
//! ```
//!
//! ### Database Compilation
//!
//! ```rust,no_run
//! use hyperscan_tokio::{DatabaseBuilder, Mode};
//!
//! // Block mode (for complete data)
//! let block_db = DatabaseBuilder::new()
//!     .add_pattern(pattern1)
//!     .add_pattern(pattern2)
//!     .mode(Mode::BLOCK)
//!     .build()?;
//!
//! // Stream mode (for data streams)
//! let stream_db = DatabaseBuilder::new()
//!     .add_pattern(pattern)
//!     .mode(Mode::STREAM)
//!     .build()?;
//!
//! // Vectored mode (for scattered data)
//! let vectored_db = DatabaseBuilder::new()
//!     .add_pattern(pattern)
//!     .mode(Mode::VECTORED)
//!     .build()?;
//! # Ok::<(), hyperscan_tokio::Error>(())
//! ```
//!
//! ## Scanning Modes
//!
//! ### Block Scanning
//!
//! For scanning complete data blocks:
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! # async fn example() -> Result<()> {
//! let scanner = Scanner::new(database)?;
//! 
//! // Scan string data
//! let matches = scanner.scan("text to scan").await?;
//!
//! // Scan bytes
//! let matches = scanner.scan_bytes(b"binary data").await?;
//!
//! // Zero-copy with Bytes
//! let data = bytes::Bytes::from_static(b"zero-copy scan");
//! let matches = scanner.scan_bytes(data).await?;
//! # Ok(())
//! # }
//! ```
//!
//! ### Stream Scanning
//!
//! For scanning data streams:
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! # async fn example() -> Result<()> {
//! let scanner = StreamScanner::new(database)?;
//!
//! // Scan a Tokio stream
//! let stream = tokio::fs::File::open("large_file.txt").await?;
//! let mut match_stream = scanner.scan_stream(stream).await?;
//!
//! while let Some(m) = match_stream.next().await {
//!     let match_result = m?;
//!     println!("Found match: {:?}", match_result);
//! }
//! # Ok(())
//! # }
//! ```
//!
//! ### Vectored Scanning
//!
//! For scanning multiple non-contiguous buffers:
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! # async fn example() -> Result<()> {
//! let scanner = VectoredScanner::new(database)?;
//!
//! let buffers = vec![
//!     b"first buffer",
//!     b"second buffer",
//!     b"third buffer",
//! ];
//!
//! let matches = scanner.scan_vectored(&buffers).await?;
//! # Ok(())
//! # }
//! ```
//!
//! ## Advanced Features
//!
//! ### Worker Pool for Parallel Processing
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! # async fn example() -> Result<()> {
//! let pool = WorkerPoolBuilder::default()
//!     .num_workers(8)
//!     .queue_size(10_000)
//!     .build(database)?;
//!
//! // Process many items in parallel
//! let jobs: Vec<ScanJob> = data_items.into_iter()
//!     .map(|data| ScanJob { id: generate_id(), data })
//!     .collect();
//!
//! let results = pool.scan_batch(jobs).await?;
//! # Ok(())
//! # }
//! ```
//!
//! ### Hot-Reloadable Patterns
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! # async fn example() -> Result<()> {
//! let reloadable = ReloadableDatabase::new(initial_database);
//!
//! // In another task, reload patterns without stopping
//! tokio::spawn(async move {
//!     let new_db = load_new_patterns().await?;
//!     reloadable.reload(new_db).await?;
//! });
//!
//! // Scanning continues with new patterns automatically
//! let scanner = Scanner::new(reloadable.current())?;
//! # Ok(())
//! # }
//! ```
//!
//! ### Database Caching
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! # async fn example() -> Result<()> {
//! let cache = DatabaseCache::builder()
//!     .max_size(100)
//!     .ttl(Duration::from_secs(3600))
//!     .build();
//!
//! // Patterns are compiled only once and cached
//! let db = cache.get_or_compile(patterns, || {
//!     DatabaseBuilder::new()
//!         .patterns(patterns)
//!         .build()
//! }).await?;
//! # Ok(())
//! # }
//! ```
//!
//! ## Error Handling
//!
//! All operations return `Result<T, Error>` with detailed error information:
//!
//! ```rust,no_run
//! # use hyperscan_tokio::prelude::*;
//! match Pattern::new("[invalid").build() {
//!     Ok(pattern) => { /* use pattern */ },
//!     Err(Error::Compile { message, pattern_id, position }) => {
//!         println!("Compile error in pattern {}: {} at position {:?}", 
//!                  pattern_id.unwrap_or(0), message, position);
//!     },
//!     Err(e) => println!("Other error: {}", e),
//! }
//! ```
//!
//! ## Thread Safety
//!
//! - `Database`: `Send + Sync` - can be shared across threads
//! - `Scanner`: `Send + Clone` - can be cloned for each thread
//! - `Scratch`: Thread-local - each thread needs its own
//!
//! ## Memory Management
//!
//! The library supports custom allocators for optimal performance:
//!
//! ```toml
//! # Use jemalloc (default)
//! hyperscan-tokio = { version = "0.1", features = ["jemalloc"] }
//!
//! # Or use mimalloc
//! hyperscan-tokio = { version = "0.1", features = ["mimalloc"] }
//! ```

#![warn(missing_docs)]
#![warn(rustdoc::broken_intra_doc_links)]
#![cfg_attr(docsrs, feature(doc_cfg))]

pub mod allocator;
pub mod builder;
pub mod cache;

/// Chimera - PCRE-compatible pattern matching with capture groups
#[cfg(feature = "chimera")]
#[cfg_attr(docsrs, doc(cfg(feature = "chimera")))]
pub mod chimera;

#[cfg(feature = "chimera")]
mod chimera_analyzer;
#[cfg(feature = "chimera")]
mod chimera_builder;
#[cfg(feature = "chimera")]
mod chimera_scanner;
#[cfg(feature = "chimera")]
mod chimera_scratch;

pub mod database;
pub mod error;
pub mod expression;
pub mod features;
pub mod literal;
pub mod pattern;
pub mod scanner;
pub mod scratch_pool;
pub mod stream;
pub mod vectored;
pub mod worker_pool;
pub mod zero_copy;

/// Metrics and observability support
#[cfg(feature = "metrics")]
#[cfg_attr(docsrs, doc(cfg(feature = "metrics")))]
pub mod metrics;

// Public exports
pub use builder::{DatabaseBuilder, Pattern, PatternBuilder};
pub use cache::{DatabaseCache, DatabaseCacheBuilder, CacheKey, CacheStats};
#[cfg(feature = "chimera")]
pub use chimera::{Chimera, Builder as ChimeraBuilder, Match as ChimeraMatch, ErrorEventType, MatchControl};
pub use pattern::{CaptureGroup, PatternInfo};
pub use database::{Database, DatabaseInfo, ExpressionInfo, ReloadableDatabase};
pub use error::{Error, Result};
pub use expression::{ExpressionContext, ExpressionContextBuilder};
pub use literal::{Literal, LiteralBuilder, LiteralFlags, LiteralDatabaseBuilder};
pub use scanner::{Match, Scanner, BlockScanner};
pub use scratch_pool::{Scratch, ScratchPool};
pub use stream::{StreamScanner, StreamState, MatchStream};
pub use vectored::VectoredScanner;
pub use worker_pool::{WorkerPool, WorkerPoolBuilder, ScanJob, ScanResult};
pub use zero_copy::ScanInput;

#[cfg(feature = "metrics")]
pub use metrics::{Metrics, MetricsCollector};

// Re-export commonly used types from sys crate
pub use hyperscan_tokio_sys::{Flags, Mode, Platform};

// Re-export feature detection
pub use features::{supported_features, SupportedFeatures, CpuArchitecture};

/// Prelude module for convenient imports
///
/// # Example
///
/// ```rust
/// use hyperscan_tokio::prelude::*;
/// ```
pub mod prelude {
    pub use crate::{
        DatabaseBuilder, Pattern, PatternBuilder,
        Database, DatabaseInfo, ExpressionInfo, ReloadableDatabase,
        Scanner, BlockScanner, Match,
        StreamScanner, VectoredScanner,
        WorkerPool, WorkerPoolBuilder,
        Literal, LiteralBuilder, LiteralDatabaseBuilder,
        Scratch, ScratchPool,
        CaptureGroup,
        ExpressionContext, ExpressionContextBuilder,
        Error, Result,
        Flags, Mode, Platform,
        supported_features,
    };
    
    #[cfg(feature = "chimera")]
    pub use crate::{
        Chimera, ChimeraBuilder, ChimeraMatch,
        ErrorEventType, MatchControl,
    };
    
    #[cfg(feature = "metrics")]
    pub use crate::{Metrics, MetricsCollector};
}