fast_yaml_parallel/
lib.rs

1//! fast-yaml-parallel: Multi-threaded YAML processing.
2//!
3//! This crate provides parallel parsing for multi-document YAML streams,
4//! leveraging Rayon for work-stealing parallelism.
5//!
6//! # Performance
7//!
8//! Expected speedup on multi-document files:
9//! - 4 cores: 3-3.5x faster
10//! - 8 cores: 6-6.5x faster
11//! - 16 cores: 10-12x faster
12//!
13//! # When to Use
14//!
15//! Use parallel processing when:
16//! - Processing multi-document YAML streams (logs, configs, data dumps)
17//! - Input size > 1MB with multiple documents
18//! - Running on multi-core hardware (4+ cores recommended)
19//!
20//! Use sequential processing when:
21//! - Single document files
22//! - Small files (<100KB)
23//! - Memory constrained environments
24//!
25//! # Examples
26//!
27//! Basic usage:
28//!
29//! ```
30//! use fast_yaml_parallel::parse_parallel;
31//!
32//! let yaml = "---\nfoo: 1\n---\nbar: 2\n---\nbaz: 3";
33//! let docs = parse_parallel(yaml)?;
34//! assert_eq!(docs.len(), 3);
35//! # Ok::<(), Box<dyn std::error::Error>>(())
36//! ```
37//!
38//! Custom configuration:
39//!
40//! ```
41//! use fast_yaml_parallel::{parse_parallel_with_config, ParallelConfig};
42//!
43//! let config = ParallelConfig::new()
44//!     .with_thread_count(Some(8))
45//!     .with_min_chunk_size(2048);
46//!
47//! let yaml = "---\nfoo: 1\n---\nbar: 2";
48//! let docs = parse_parallel_with_config(yaml, &config)?;
49//! # Ok::<(), Box<dyn std::error::Error>>(())
50//! ```
51
52#![forbid(unsafe_code)]
53#![warn(missing_docs)]
54
55mod chunker;
56mod config;
57mod error;
58mod processor;
59
60pub use config::ParallelConfig;
61pub use error::{ParallelError, Result};
62pub use fast_yaml_core::Value;
63
64/// Parse multi-document YAML stream in parallel.
65///
66/// Automatically detects document boundaries and distributes
67/// parsing across multiple threads. Falls back to sequential
68/// parsing for single-document inputs.
69///
70/// # Errors
71///
72/// Returns `ParallelError::ParseError` if any document fails to parse.
73/// The error includes the document index for debugging.
74///
75/// # Examples
76///
77/// ```
78/// use fast_yaml_parallel::parse_parallel;
79///
80/// let yaml = "---\nfoo: 1\n---\nbar: 2";
81/// let docs = parse_parallel(yaml)?;
82/// assert_eq!(docs.len(), 2);
83/// # Ok::<(), Box<dyn std::error::Error>>(())
84/// ```
85pub fn parse_parallel(input: &str) -> Result<Vec<Value>> {
86    let config = ParallelConfig::default();
87    processor::process_parallel(input, &config)
88}
89
90/// Parse multi-document YAML with custom configuration.
91///
92/// Allows fine-tuning of parallelism parameters for specific
93/// workloads and hardware configurations.
94///
95/// # Errors
96///
97/// Returns `ParallelError` if parsing or configuration fails.
98///
99/// # Examples
100///
101/// ```
102/// use fast_yaml_parallel::{parse_parallel_with_config, ParallelConfig};
103///
104/// let config = ParallelConfig::new()
105///     .with_thread_count(Some(4));
106///
107/// let yaml = "---\nfoo: 1\n---\nbar: 2";
108/// let docs = parse_parallel_with_config(yaml, &config)?;
109/// # Ok::<(), Box<dyn std::error::Error>>(())
110/// ```
111pub fn parse_parallel_with_config(input: &str, config: &ParallelConfig) -> Result<Vec<Value>> {
112    processor::process_parallel(input, config)
113}
114
115#[cfg(test)]
116mod tests {
117    use super::*;
118
119    #[test]
120    fn test_parse_parallel_basic() {
121        let yaml = "---\nfoo: 1\n---\nbar: 2";
122        let docs = parse_parallel(yaml).unwrap();
123        assert_eq!(docs.len(), 2);
124    }
125
126    #[test]
127    fn test_parse_parallel_single_document() {
128        let yaml = "foo: 1\nbar: 2";
129        let docs = parse_parallel(yaml).unwrap();
130        assert_eq!(docs.len(), 1);
131    }
132
133    #[test]
134    fn test_parse_parallel_error() {
135        let yaml = "---\nvalid: true\n---\ninvalid: [";
136        let result = parse_parallel(yaml);
137        assert!(result.is_err());
138    }
139
140    #[test]
141    fn test_parse_parallel_with_config() {
142        let config = ParallelConfig::new().with_thread_count(Some(2));
143        let yaml = "---\nfoo: 1\n---\nbar: 2";
144        let docs = parse_parallel_with_config(yaml, &config).unwrap();
145        assert_eq!(docs.len(), 2);
146    }
147
148    #[test]
149    fn test_parse_parallel_empty_input() {
150        let yaml = "";
151        let docs = parse_parallel(yaml).unwrap();
152        assert_eq!(docs.len(), 0);
153    }
154
155    #[test]
156    fn test_parse_parallel_many_documents() {
157        use std::fmt::Write;
158        let mut yaml = String::new();
159        for i in 0..100 {
160            yaml.push_str("---\n");
161            let _ = writeln!(yaml, "id: {i}");
162        }
163
164        let docs = parse_parallel(&yaml).unwrap();
165        assert_eq!(docs.len(), 100);
166    }
167}
fast_yaml_parallel/lib.rs

fast_yaml_parallel/
lib.rs
