Trait StreamingBatchOperation

Source

pub trait StreamingBatchOperation: Send + Sync {
    type Output: Send;

    // Required methods
    fn process_file_streaming(
        &self,
        path: &Path,
    ) -> Result<Self::Output, CliError>;
    fn name(&self) -> &str;

    // Provided method
    fn supports_streaming(&self) -> bool { ... }
}

Expand description

Trait for streaming batch operations on HEDL files.

Unlike BatchOperation which loads entire files into memory, streaming operations process files incrementally with constant memory usage. This is ideal for processing large files (>100MB) or when memory is constrained.

§Memory Characteristics

Standard operations: O(num_threads × file_size)
Streaming operations: O(buffer_size + ID_set) ≈ constant

§Type Parameters

Output - The type returned on successful processing of a file

§Examples

use hedl_cli::batch::StreamingBatchOperation;
use hedl_cli::error::CliError;
use std::path::Path;

struct StreamingCountOperation;

impl StreamingBatchOperation for StreamingCountOperation {
    type Output = usize;

    fn process_file_streaming(&self, path: &Path) -> Result<Self::Output, CliError> {
        use std::io::BufReader;
        use std::fs::File;
        use hedl_stream::StreamingParser;

        let file = File::open(path).map_err(|e| CliError::io_error(path, e))?;
        let reader = BufReader::new(file);
        let parser = StreamingParser::new(reader)
            .map_err(|e| CliError::parse(e.to_string()))?;

        let count = parser.filter(|e| {
            matches!(e, Ok(hedl_stream::NodeEvent::Node(_)))
        }).count();

        Ok(count)
    }

    fn name(&self) -> &str {
        "count-streaming"
    }
}