async_regex/

lib.rs

//! # async-regex
//! 
//! **Empower regex with streaming capabilities!**
//! 
//! This crate provides high-performance streaming pattern search that leverages the power of regex
//! for pattern matching while adding streaming capabilities for processing large data streams
//! without loading everything into memory.
//! 
//! ## Key Features
//! 
//! - **Regex-powered pattern matching**: Uses the robust `regex` crate for pattern matching
//! - **Streaming support**: Process data as it arrives without loading everything into memory
//! - **Async and sync APIs**: Both async and synchronous versions available
//! - **Multi-byte pattern support**: Unlike standard `read_until` which only supports single bytes
//! - **Memory efficient**: Minimal memory footprint with zero-copy operations where possible
//! 
//! ## Why This Crate?
//! 
//! The standard `std::io::BufRead::read_until` and `tokio::io::AsyncBufRead::read_until` methods
//! only support single-byte delimiters. This crate extends that functionality to support
//! multi-byte patterns using regex, making it perfect for parsing protocols, log files,
//! and other structured data streams.

use std::io;
use std::io::BufRead;
use std::pin::Pin;

use futures::{AsyncBufRead, AsyncBufReadExt};
use regex::bytes::Regex;

/// Carriage Return + Line Feed constant for convenience
pub const CRLF: &[u8] = &[0xd, 0xa];


/// Read from a BufRead stream until a regex pattern is found.
/// 
/// This function extends the standard `read_until` functionality to support multi-byte patterns
/// using regex. It reads data from the stream until the specified regex pattern is found,
/// storing all data up to and including the match in the provided buffer.
/// 
/// # Arguments
/// 
/// * `reader` - A BufRead stream to read from
/// * `pattern` - A regex pattern string to search for
/// * `to` - Buffer to store the read data
/// 
/// # Returns
/// 
/// Returns a tuple containing:
/// - The matched substring as bytes
/// - The total number of bytes read
/// 
/// # Examples
/// 
/// ```rust
/// use async_regex::read_until_pattern;
/// use std::io::Cursor;
/// 
/// let mut reader = Cursor::new(b"hello world test pattern");
/// let mut buffer = Vec::new();
/// 
/// let (matched, size) = read_until_pattern(&mut reader, r"\w+", &mut buffer).unwrap();
/// assert_eq!(matched, b"hello");
/// assert_eq!(buffer, b"hello");
/// ```
pub fn read_until_pattern(
    reader: &mut impl BufRead,
    pattern: &str,
    to: &mut Vec<u8>,
) -> io::Result<(Vec<u8>, usize)> {
    if pattern.is_empty() {
        return Ok((Vec::new(), 0));
    }
    let regex = Regex::new(pattern).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
    let original_len = to.len();
    let mut search_buffer = Vec::new();

    loop {
        let (found_pattern, consume_amount, matched_substring) = {
            let available = reader.fill_buf()?;
            if available.is_empty() {
                (false, 0, Vec::new())
            } else {
                let available_len = available.len();
                search_buffer.extend_from_slice(available);

                if let Some((pos, match_len)) = find_pattern(&search_buffer, &regex) {
                    let end_pos = pos + match_len;
                    to.extend_from_slice(&search_buffer[..end_pos]);
                    
                    let matched = search_buffer[pos..end_pos].to_vec();
                    let consume_from_current =
                        end_pos.saturating_sub(search_buffer.len() - available_len);
                    (true, consume_from_current, matched)
                } else {
                    let keep_size = (pattern.len() - 1).min(search_buffer.len());

                    if search_buffer.len() > keep_size {
                        let move_to_output = search_buffer.len() - keep_size;
                        to.extend_from_slice(&search_buffer[..move_to_output]);
                        search_buffer.drain(..move_to_output);
                    }

                    (false, available_len, Vec::new())
                }
            }
        };

        if found_pattern {
            reader.consume(consume_amount);
            return Ok((matched_substring, to.len() - original_len));
        }

        if consume_amount == 0 {
            to.extend_from_slice(&search_buffer);
            break;
        }

        reader.consume(consume_amount);
    }

    Ok((Vec::new(), to.len() - original_len))
}

/// Find a regex pattern in a byte slice.
/// 
/// This is a utility function that uses regex to find patterns in byte data.
/// It's used internally by the streaming functions but can also be used directly
/// for in-memory pattern matching.
/// 
/// # Arguments
/// 
/// * `haystack` - The byte slice to search in
/// * `needle` - The compiled regex pattern to search for
/// 
/// # Returns
/// 
/// Returns `Some((start, length))` if the pattern is found, `None` otherwise.
/// 
/// # Examples
/// 
/// ```rust
/// use async_regex::find_pattern;
/// use regex::bytes::Regex;
/// 
/// let regex = Regex::new(r"\w+").unwrap();
/// let data = b"hello world";
/// 
/// if let Some((start, len)) = find_pattern(data, &regex) {
///     println!("Found pattern at position {} with length {}", start, len);
/// }
/// ```
#[inline]
pub fn find_pattern(haystack: &[u8], needle: &Regex) -> Option<(usize, usize)> {
    needle.find(haystack).map(|m| (m.start(), m.len()))
}

/// Async version of `read_until_pattern` for streaming data.
/// 
/// This function provides the same functionality as `read_until_pattern` but works with
/// async streams, making it perfect for processing network data, file streams, or any
/// other async I/O operations.
/// 
/// # Arguments
/// 
/// * `reader` - An AsyncBufRead stream to read from
/// * `pattern` - A regex pattern string to search for
/// * `to` - Buffer to store the read data
/// 
/// # Returns
/// 
/// Returns a tuple containing:
/// - The matched substring as bytes
/// - The total number of bytes read
/// 
/// # Examples
/// 
/// ```rust
/// use async_regex::read_until_pattern_async;
/// use futures::io::Cursor;
/// use tokio::runtime::Runtime;
/// 
/// let rt = Runtime::new().unwrap();
/// rt.block_on(async {
///     let mut reader = Cursor::new(b"hello world test pattern");
///     let mut buffer = Vec::new();
/// 
///     let (matched, size) = read_until_pattern_async(&mut reader, r"\w+", &mut buffer).await.unwrap();
///     assert_eq!(matched, b"hello");
///     assert_eq!(buffer, b"hello");
/// });
/// ```
pub async fn read_until_pattern_async<R: AsyncBufRead + Unpin>(
    reader: &mut R,
    pattern: &str,
    to: &mut Vec<u8>,
) -> io::Result<(Vec<u8>, usize)> {
    if pattern.is_empty() {
        return Ok((Vec::new(), 0));
    }
    let regex = Regex::new(pattern).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
    let original_len = to.len();
    let mut search_buffer = Vec::new();

    loop {
        let (found_pattern, consume_amount, matched_substring) = {
            let available = reader.fill_buf().await?;
            if available.is_empty() {
                (false, 0, Vec::new())
            } else {
                let available_len = available.len();

                search_buffer.extend_from_slice(available);

                if let Some((pos, match_len)) = find_pattern(&search_buffer, &regex) {
                    let end_pos = pos + match_len;
                    to.extend_from_slice(&search_buffer[..end_pos]);
                    
                    // Extract the matched substring
                    let matched = search_buffer[pos..end_pos].to_vec();

                    let consume_from_current =
                        end_pos.saturating_sub(search_buffer.len() - available_len);
                    (true, consume_from_current, matched)
                } else {
                    let keep_size = (pattern.len() - 1).min(search_buffer.len());

                    if search_buffer.len() > keep_size {
                        let move_to_output = search_buffer.len() - keep_size;
                        to.extend_from_slice(&search_buffer[..move_to_output]);
                        search_buffer.drain(..move_to_output);
                    }

                    (false, available_len, Vec::new())
                }
            }
        };

        if found_pattern {
            Pin::new(&mut *reader).consume(consume_amount);
            return Ok((matched_substring, to.len() - original_len));
        }

        if consume_amount == 0 {
            to.extend_from_slice(&search_buffer);
            break;
        }

        Pin::new(&mut *reader).consume(consume_amount);
    }

    Ok((Vec::new(), to.len() - original_len))
}


/// Read from a BufRead stream while any of the specified bytes are found.
/// 
/// This function reads data from the stream while the current byte matches any byte
/// in the provided check set. It stops when it encounters a byte not in the set.
/// 
/// # Arguments
/// 
/// * `reader` - A BufRead stream to read from
/// * `check_set` - A slice of bytes to match against
/// * `to` - Buffer to store the read data
/// 
/// # Returns
/// 
/// Returns a tuple containing:
/// - The first byte that didn't match (the stop byte)
/// - The number of bytes read
/// 
/// # Examples
/// 
/// ```rust
/// use async_regex::read_while_any;
/// use std::io::Cursor;
/// 
/// let mut reader = Cursor::new(b"aaaaabbbccc");
/// let mut buffer = Vec::new();
/// let check_set = b"ab";
/// 
/// let (stop_byte, count) = read_while_any(&mut reader, check_set, &mut buffer).unwrap();
/// assert_eq!(buffer, b"aaaaabbb");
/// assert_eq!(stop_byte, b'c');
/// ```
pub fn read_while_any(
    reader: &mut impl BufRead,
    check_set: &[u8],
    to: &mut Vec<u8>,
) -> io::Result<(u8, usize)> {
    if check_set.is_empty() {
        let available = reader.fill_buf()?;
        if available.is_empty() {
            return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "EOF reached"));
        }
        let first_byte = available[0];
        reader.consume(1);
        return Ok((first_byte, 0));
    }

    let original_len = to.len();

    let mut lookup = [false; 256];
    for &byte in check_set {
        lookup[byte as usize] = true;
    }

    loop {
        let (stop_byte, consume_amount) = {
            let available = reader.fill_buf()?;
            if available.is_empty() {
                return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "EOF reached"));
            }
            let mut pos = 0;
            while pos < available.len() {
                let byte = available[pos];
                if !lookup[byte as usize] {
                    to.extend_from_slice(&available[..pos]);
                    break;
                }
                pos += 1;
            }

            if pos < available.len() {
                (Some(available[pos]), pos)
            } else {
                to.extend_from_slice(available);
                (None, available.len())
            }
        };

        reader.consume(consume_amount);

        if let Some(byte) = stop_byte {
            return Ok((byte, to.len() - original_len));
        }
    }
}

/// Async version of `read_while_any` for streaming data.
/// 
/// This function provides the same functionality as `read_while_any` but works with
/// async streams, making it perfect for processing network data, file streams, or any
/// other async I/O operations.
/// 
/// # Arguments
/// 
/// * `reader` - An AsyncBufRead stream to read from
/// * `check_set` - A slice of bytes to match against
/// * `to` - Buffer to store the read data
/// 
/// # Returns
/// 
/// Returns a tuple containing:
/// - The first byte that didn't match (the stop byte)
/// - The number of bytes read
/// 
/// # Examples
/// 
/// ```rust
/// use async_regex::read_while_any_async;
/// use futures::io::Cursor;
/// use tokio::runtime::Runtime;
/// 
/// let rt = Runtime::new().unwrap();
/// rt.block_on(async {
///     let mut reader = Cursor::new(b"aaaaabbbccc");
///     let mut buffer = Vec::new();
///     let check_set = b"ab";
/// 
///     let (stop_byte, count) = read_while_any_async(&mut reader, check_set, &mut buffer).await.unwrap();
///     assert_eq!(buffer, b"aaaaabbb");
///     assert_eq!(stop_byte, b'c');
/// });
/// ```
pub async fn read_while_any_async<R: AsyncBufRead + Unpin>(
    reader: &mut R,
    check_set: &[u8],
    to: &mut Vec<u8>,
) -> io::Result<(u8, usize)> {
    if check_set.is_empty() {
        let available = reader.fill_buf().await?;
        if available.is_empty() {
            return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "EOF reached"));
        }
        let first_byte = available[0];
        Pin::new(&mut *reader).consume(1);
        return Ok((first_byte, 0));
    }

    let original_len = to.len();
    let mut lookup = [false; 256];
    for &byte in check_set {
        lookup[byte as usize] = true;
    }

    loop {
        let (stop_byte, consume_amount) = {
            let available = reader.fill_buf().await?;
            if available.is_empty() {
                return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "EOF reached"));
            }
            let mut pos = 0;
            while pos < available.len() {
                let byte = available[pos];
                if !lookup[byte as usize] {
                    to.extend_from_slice(&available[..pos]);
                    break;
                }
                pos += 1;
            }

            if pos < available.len() {
                (Some(available[pos]), pos)
            } else {
                to.extend_from_slice(available);
                (None, available.len())
            }
        };

        Pin::new(&mut *reader).consume(consume_amount);

        if let Some(byte) = stop_byte {
            return Ok((byte, to.len() - original_len));
        }
    }
}

#[cfg(test)]
mod tests {
    use futures::executor::block_on;

    use super::*;
    use std::io;
    use std::io::Cursor;
    use std::str;
    pub const CDATA_TAG: &str = "![CDATA[";
    pub const COMMENT_TAG: &str = "!--";

    #[test]
    fn test_read_until_pattern() -> io::Result<()> {
        let mut string = "<a b:b1 c:c1 d:d1 />\n<a b:b2 c:c2 d:d2 />".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "c:c2", &mut to)?;
        str::from_utf8(&to).unwrap();
        assert_eq!(
            str::from_utf8(&to).unwrap(),
            "<a b:b1 c:c1 d:d1 />\n<a b:b2 c:c2"
        );
        assert_eq!(size, "<a b:b1 c:c1 d:d1 />\n<a b:b2 c:c2".len());
        assert_eq!(matched, "c:c2".as_bytes());

        let mut string = "<a b:b1 c:c1 d:d1 />\n<anode b:b2 c:c2 d:d2 />".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "<anode", &mut to)?;
        assert_eq!(str::from_utf8(&to).unwrap(), "<a b:b1 c:c1 d:d1 />\n<anode");
        assert_eq!(size, "<a b:b1 c:c1 d:d1 />\n<anode".len());
        assert_eq!(matched, "<anode".as_bytes());

        let mut string = "<a b:b1 c:c1 d:d1 />\n<anode b:b2 c:c2 d:d2 />".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "<bnode", &mut to)?;
        assert_eq!(string, "".as_bytes());
        assert_eq!(
            to,
            "<a b:b1 c:c1 d:d1 />\n<anode b:b2 c:c2 d:d2 />".as_bytes()
        );
        assert_eq!(size, 45);
        assert_eq!(matched, Vec::<u8>::new());

        let mut string = "1-x-xx2".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "-xx", &mut to)?;
        assert_eq!(string, "2".as_bytes());
        assert_eq!(to, "1-x-xx".as_bytes());
        assert_eq!(size, 6);
        assert_eq!(matched, "-xx".as_bytes());

        let mut string = "$1131132$".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "1132", &mut to)?;
        assert_eq!(string, "$".as_bytes());
        assert_eq!(to, "$1131132".as_bytes());
        assert_eq!(size, 8);
        assert_eq!(matched, "1132".as_bytes());

        let mut string = "12".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "13", &mut to)?;
        assert_eq!(string, "".as_bytes());
        assert_eq!(to, "12".as_bytes());
        assert_eq!(size, 2);
        assert_eq!(matched, Vec::<u8>::new());

        let mut string = "222222".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "33333333", &mut to)?;
        assert_eq!(string, "".as_bytes());
        assert_eq!(to, "222222".as_bytes());
        assert_eq!(size, 6);
        assert_eq!(matched, Vec::<u8>::new());

        Ok(())
    }

    #[test]
    fn test_read_until_pattern_async() -> io::Result<()> {
        block_on(async {
            let mut string = "<a b:b1 c:c1 d:d1 />\n<a b:b2 c:c2 d:d2 />".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "c:c2", &mut to)
                .await
                .unwrap();
            str::from_utf8(&to).unwrap();
            assert_eq!(
                str::from_utf8(&to).unwrap(),
                "<a b:b1 c:c1 d:d1 />\n<a b:b2 c:c2"
            );
            assert_eq!(size, "<a b:b1 c:c1 d:d1 />\n<a b:b2 c:c2".len());
            assert_eq!(matched, "c:c2".as_bytes());

            let mut string = "<a b:b1 c:c1 d:d1 />\n<anode b:b2 c:c2 d:d2 />".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "<anode", &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&to).unwrap(), "<a b:b1 c:c1 d:d1 />\n<anode");
            assert_eq!(size, "<a b:b1 c:c1 d:d1 />\n<anode".len());
            assert_eq!(matched, "<anode".as_bytes());

            let mut string = "<a b:b1 c:c1 d:d1 />\n<anode b:b2 c:c2 d:d2 />".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "<bnode", &mut to)
                .await
                .unwrap();
            assert_eq!(string, "".as_bytes());
            assert_eq!(
                to,
                "<a b:b1 c:c1 d:d1 />\n<anode b:b2 c:c2 d:d2 />".as_bytes()
            );
            assert_eq!(size, 45);
            assert_eq!(matched, Vec::<u8>::new());

            let mut string = "1-x-xx2".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "-xx", &mut to)
                .await
                .unwrap();
            assert_eq!(string, "2".as_bytes());
            assert_eq!(to, "1-x-xx".as_bytes());
            assert_eq!(size, 6);
            assert_eq!(matched, "-xx".as_bytes());

            let mut string = "$1131132$".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "1132", &mut to)
                .await
                .unwrap();
            assert_eq!(string, "$".as_bytes());
            assert_eq!(to, "$1131132".as_bytes());
            assert_eq!(size, 8);
            assert_eq!(matched, "1132".as_bytes());

            let mut string = "12".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "13", &mut to)
                .await
                .unwrap();
            assert_eq!(string, "".as_bytes());
            assert_eq!(to, "12".as_bytes());
            assert_eq!(size, 2);
            assert_eq!(matched, Vec::<u8>::new());

            let mut string = "222222".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "33333333", &mut to)
                .await
                .unwrap();
            assert_eq!(string, "".as_bytes());
            assert_eq!(to, "222222".as_bytes());
            assert_eq!(size, 6);
            assert_eq!(matched, Vec::<u8>::new());
        });

        Ok(())
    }

    #[test]
    fn test_read_until_pattern_matched_substring() -> io::Result<()> {
        let mut string = "hello world test pattern".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, r"\w+", &mut to)?;
        assert_eq!(str::from_utf8(&to).unwrap(), "hello");
        assert_eq!(size, 5);
        assert_eq!(matched, "hello".as_bytes());

        let mut string = "email@example.com and more text".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, r"\w+@\w+\.\w+", &mut to)?;
        assert_eq!(str::from_utf8(&to).unwrap(), "email@example.com");
        assert_eq!(size, 17);
        assert_eq!(matched, "email@example.com".as_bytes());

        let mut string = "no match here".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "xyz", &mut to)?;
        assert_eq!(str::from_utf8(&to).unwrap(), "no match here");
        assert_eq!(size, 13);
        assert_eq!(matched, Vec::<u8>::new());

        let mut string = "some text".as_bytes();
        let mut to = Vec::new();
        let (matched, size) = read_until_pattern(&mut string, "", &mut to)?;
        assert_eq!(size, 0);
        assert_eq!(matched, Vec::<u8>::new());

        Ok(())
    }

    #[test]
    fn test_read_until_pattern_async_matched_substring() -> io::Result<()> {
        block_on(async {
            // Test with regex pattern that matches multiple characters
            let mut string = "hello world test pattern".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, r"\w+", &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&to).unwrap(), "hello");
            assert_eq!(size, 5);
            assert_eq!(matched, "hello".as_bytes());

            // Test with complex regex pattern
            let mut string = "email@example.com and more text".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, r"\w+@\w+\.\w+", &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&to).unwrap(), "email@example.com");
            assert_eq!(size, 17);
            assert_eq!(matched, "email@example.com".as_bytes());

            // Test with pattern that doesn't match
            let mut string = "no match here".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "xyz", &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&to).unwrap(), "no match here");
            assert_eq!(size, 13);
            assert_eq!(matched, Vec::<u8>::new());

            // Test with empty pattern
            let mut string = "some text".as_bytes();
            let mut to = Vec::new();
            let (matched, size) = read_until_pattern_async(&mut string, "", &mut to)
                .await
                .unwrap();
            assert_eq!(size, 0);
            assert_eq!(matched, Vec::<u8>::new());
        });

        Ok(())
    }

    #[test]
    fn test_read_while_any() -> io::Result<()> {
        let mut string = b"aaaaabbbccc" as &[u8];
        let mut to = Vec::new();
        let up_to = [b"a"[0], b"b"[0]];
        let (byte, _) = read_while_any(&mut string, &up_to, &mut to)?;
        assert_eq!(str::from_utf8(&to).unwrap(), "aaaaabbb");
        assert_eq!(byte, "c".as_bytes()[0]);

        let mut data = b"12345abc" as &[u8];
        let mut result = Vec::new();
        let digits = b"0123456789";
        let (stop_byte, count) = read_while_any(&mut data, digits, &mut result)?;
        assert_eq!(result, b"12345");
        assert_eq!(stop_byte, b'a');
        assert_eq!(count, 5);

        let data = b"   \t\n  text";
        let mut reader = Cursor::new(data);
        let mut result = Vec::new();
        let whitespace = b" \t\n\r";
        let (stop_byte, count) = read_while_any(&mut reader, whitespace, &mut result)?;
        assert_eq!(result, b"   \t\n  ");
        assert_eq!(stop_byte, b't');
        assert_eq!(count, 7);

        let data = b"hello";
        let mut reader = Cursor::new(data);
        let mut result = Vec::new();
        let empty_set = b"";
        let (stop_byte, count) = read_while_any(&mut reader, empty_set, &mut result)?;
        assert_eq!(result.len(), 0);
        assert_eq!(stop_byte, b'h');
        assert_eq!(count, 0);

        Ok(())
    }

    #[test]
    fn test_read_while_any_async() -> io::Result<()> {
        block_on(async {
            let mut string = b"aaaaabbbccc" as &[u8];
            let mut to = Vec::new();
            let up_to = [b"a"[0], b"b"[0]];
            let (byte, _) = read_while_any_async(&mut string, &up_to, &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&to).unwrap(), "aaaaabbb");
            assert_eq!(byte, "c".as_bytes()[0]);

            let mut data = b"12345abc" as &[u8];
            let mut result = Vec::new();
            let digits = b"0123456789";
            let (stop_byte, count) = read_while_any_async(&mut data, digits, &mut result)
                .await
                .unwrap();
            assert_eq!(result, b"12345");
            assert_eq!(stop_byte, b'a');
            assert_eq!(count, 5);

            let mut data = b"   \t\n  text" as &[u8];
            let mut result = Vec::new();
            let whitespace = b" \t\n\r";
            let (stop_byte, count) = read_while_any_async(&mut data, whitespace, &mut result)
                .await
                .unwrap();
            assert_eq!(result, b"   \t\n  ");
            assert_eq!(stop_byte, b't');
            assert_eq!(count, 7);

            let mut data = b"hello" as &[u8];
            let mut result = Vec::new();
            let empty_set = b"";
            let (stop_byte, count) = read_while_any_async(&mut data, empty_set, &mut result)
                .await
                .unwrap();
            assert_eq!(result.len(), 0);
            assert_eq!(stop_byte, b'h');
            assert_eq!(count, 0);
        });

        Ok(())
    }

    #[test]
    fn test_read_until_any() -> io::Result<()> {
        let mut string = b"123456789" as &[u8];
        let mut to = Vec::new();
        let check_set = "[43]";
        let (sep, _) = read_until_pattern(&mut string, &check_set, &mut to)?;
        assert_eq!(str::from_utf8(&to).unwrap(), "123");
        assert_eq!(String::from_utf8(sep).unwrap(), "3");

        let mut data = b"abc123def" as &[u8];
        let mut result = Vec::new();
        let digits_pattern = "[0-9]";
        let (found_pattern, count) = read_until_pattern(&mut data, digits_pattern, &mut result)?;
        assert_eq!(result, b"abc1");
        assert_eq!(found_pattern, b"1");
        assert_eq!(count, 4);
        assert_eq!(data, b"23def");

        let mut data = b"hello world!" as &[u8];
        let mut result = Vec::new();
        let punctuation_pattern = r"[!@#$%^&*(),.?]";
        let (found_pattern, count) = read_until_pattern(&mut data, punctuation_pattern, &mut result)?;
        assert_eq!(result, b"hello world!");
        assert_eq!(found_pattern, b"!");
        assert_eq!(count, 12);
        assert_eq!(data, b"");

        let mut data = b"!hello" as &[u8];
        let mut result = Vec::new();
        let punctuation_pattern = r"!";
        let (found_pattern, count) = read_until_pattern(&mut data, punctuation_pattern, &mut result)?;
        assert_eq!(result, b"!");
        assert_eq!(found_pattern, b"!");
        assert_eq!(count, 1);
        assert_eq!(data, b"hello"); // Remaining data

        let mut data = b"abc,def.ghi!" as &[u8];
        let punctuation_pattern = r"[,.!]";
        let mut result1 = Vec::new();
        let (found1, _count1) = read_until_pattern(&mut data, punctuation_pattern, &mut result1)?;
        assert_eq!(result1, b"abc,");
        assert_eq!(found1, b",");
        assert_eq!(data, b"def.ghi!");
        let mut result2 = Vec::new();
        let (found2, _count2) = read_until_pattern(&mut data, punctuation_pattern, &mut result2)?;
        assert_eq!(result2, b"def.");
        assert_eq!(found2, b".");
        assert_eq!(data, b"ghi!");
        let mut result3 = Vec::new();
        let (found3, _count3) = read_until_pattern(&mut data, punctuation_pattern, &mut result3)?;
        assert_eq!(result3, b"ghi!");
        assert_eq!(found3, b"!");
        assert_eq!(data, b"");

        let mut data = b"hello world" as &[u8];
        let mut result = Vec::new();
        let digits_pattern = "[0-9]";
        let (found_pattern, count) = read_until_pattern(&mut data, digits_pattern, &mut result)?;
        assert_eq!(result, b"hello world");
        assert_eq!(found_pattern, Vec::<u8>::new());
        assert_eq!(count, 11);
        assert_eq!(data, b"");

        let mut data = b"hello" as &[u8];
        let mut result = Vec::new();
        let empty_pattern = "";
        let (found_pattern, count) = read_until_pattern(&mut data, empty_pattern, &mut result)?;
        assert_eq!(count, 0);
        assert_eq!(found_pattern, Vec::<u8>::new());

        Ok(())
    }

    #[test]
    fn test_read_until_any_async() -> io::Result<()> {
        block_on(async {
            let mut string = b"123456789" as &[u8];
            let mut to = Vec::new();
            let check_set = "[43]";
            let (sep, _) = read_until_pattern_async(&mut string, &check_set, &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&to).unwrap(), "123");
            assert_eq!(str::from_utf8(&sep).unwrap(), "3");

            let mut data = b"abc123def" as &[u8];
            let mut result = Vec::new();
            let digits_pattern = "[0-9]";
            let (found_pattern, count) = read_until_pattern_async(&mut data, digits_pattern, &mut result)
                .await
                .unwrap();
            assert_eq!(result, b"abc1");
            assert_eq!(found_pattern, b"1");
            assert_eq!(count, 4);
            assert_eq!(data, b"23def");

            let mut data = b"hello world!" as &[u8];
            let mut result = Vec::new();
            let punctuation_pattern = r"[!@#$%^&*(),.?]";
            let (found_pattern, count) = read_until_pattern_async(&mut data, punctuation_pattern, &mut result)
                .await
                .unwrap();
            assert_eq!(result, b"hello world!");
            assert_eq!(found_pattern, b"!");
            assert_eq!(count, 12);
            assert_eq!(data, b"");

            let mut data = b"!hello" as &[u8];
            let mut result = Vec::new();
            let punctuation_pattern = r"!";
            let (found_pattern, count) = read_until_pattern_async(&mut data, punctuation_pattern, &mut result)
                .await
                .unwrap();
            assert_eq!(result, b"!");
            assert_eq!(found_pattern, b"!");
            assert_eq!(count, 1);
            assert_eq!(data, b"hello"); // Remaining data

            let mut data = b"abc,def.ghi!" as &[u8];
            let punctuation_pattern = r"[,.!]";
            let mut result1 = Vec::new();
            let (found1, _count1) = read_until_pattern_async(&mut data, punctuation_pattern, &mut result1)
                .await
                .unwrap();
            assert_eq!(result1, b"abc,");
            assert_eq!(found1, b",");
            assert_eq!(data, b"def.ghi!");
            let mut result2 = Vec::new();
            let (found2, _count2) = read_until_pattern_async(&mut data, punctuation_pattern, &mut result2)
                .await
                .unwrap();
            assert_eq!(result2, b"def.");
            assert_eq!(found2, b".");
            assert_eq!(data, b"ghi!");
            let mut result3 = Vec::new();
            let (found3, _count3) = read_until_pattern_async(&mut data, punctuation_pattern, &mut result3)
                .await
                .unwrap();
            assert_eq!(result3, b"ghi!");
            assert_eq!(found3, b"!");
            assert_eq!(data, b"");

            let mut data = b"hello world" as &[u8];
            let mut result = Vec::new();
            let digits_pattern = "[0-9]";
            let (found_pattern, count) = read_until_pattern_async(&mut data, digits_pattern, &mut result)
                .await
                .unwrap();
            assert_eq!(result, b"hello world");
            assert_eq!(found_pattern, Vec::<u8>::new());
            assert_eq!(count, 11);
            assert_eq!(data, b"");

            let mut data = b"hello" as &[u8];
            let mut result = Vec::new();
            let empty_pattern = "";
            let (found_pattern, count) = read_until_pattern_async(&mut data, empty_pattern, &mut result)
                .await
                .unwrap();
            assert_eq!(count, 0);
            assert_eq!(found_pattern, Vec::<u8>::new());
        });

        Ok(())
    }

    #[test]
    fn test_read_until_any_pattern() -> io::Result<()> {
        let mut string = "![CD!-![CDATA!--?abcd".as_bytes();
        let mut to = Vec::new();
        let pattern = format!("{}|{}|\\?", regex::escape(CDATA_TAG), regex::escape(COMMENT_TAG));
        let (matched, size) = read_until_pattern(&mut string, &pattern, &mut to)?;
        assert_eq!(str::from_utf8(&matched).unwrap(), COMMENT_TAG);
        assert_eq!(size, "![CD!-![CDATA!--".len());
        assert_eq!(str::from_utf8(&to).unwrap(), "![CD!-![CDATA!--");

        let string = r##"<div class="1" >
            <div class="2">
                <div class="3">
                    <div class="4"></div>
                </div>
            </div>
            <div class="5"></div>
        </div>
        "##;

        let tag = "div";
        let mut to = vec![];
        let open_pattern = format!("<{}", tag);
        let close_pattern = format!("</{}>", tag);
        let pattern = format!("{}|{}", regex::escape(&close_pattern), regex::escape(&open_pattern));
        let reader = &mut string.as_bytes();
        io::BufRead::read_until(reader, ">".as_bytes()[0], &mut to)?;
        let (matched, _size) = read_until_pattern(reader, &pattern, &mut to)?;
        assert_eq!(str::from_utf8(&matched).unwrap(), "<div");

        let mut string = "$1131132$".as_bytes();
        let mut to = Vec::new();
        let pattern = "1132|115";
        let (matched, size) = read_until_pattern(&mut string, pattern, &mut to)?;
        assert_eq!(string, "$".as_bytes());
        assert_eq!(to, "$1131132".as_bytes());
        assert_eq!(size, 8);
        assert_eq!(str::from_utf8(&matched).unwrap(), "1132");

        let string = "<a />\
            <b></b>\
        </link>";
        let mut to = Vec::new();
        let pattern = "</link>|<link>";
        let (matched, size) = read_until_pattern(&mut string.as_bytes(), pattern, &mut to)?;
        assert_eq!(to, string.as_bytes());
        assert_eq!(std::str::from_utf8(&matched).unwrap(), "</link>");
        assert_eq!(size, string.len());

        let mut string = "1</link>".as_bytes();
        let mut to = Vec::new();
        let pattern = "123|890";
        let (matched, size) = read_until_pattern(&mut string, pattern, &mut to)?;
        assert_eq!(to, "1</link>".as_bytes());
        assert_eq!(size, 8);
        assert_eq!(matched, Vec::<u8>::new());

        Ok(())
    }

    #[test]
    fn test_read_until_any_pattern_async() -> io::Result<()> {
        block_on(async {
            let mut string = "![CD!-![CDATA!--?abcd".as_bytes();
            let mut to = Vec::new();
            let pattern = format!("{}|{}|\\?", regex::escape(CDATA_TAG), regex::escape(COMMENT_TAG));
            let (matched, size) = read_until_pattern_async(&mut string, &pattern, &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&matched).unwrap(), COMMENT_TAG);
            assert_eq!(size, "![CD!-![CDATA!--".len());
            assert_eq!(str::from_utf8(&to).unwrap(), "![CD!-![CDATA!--");

            let string = r##"<div class="1" >
                <div class="2">
                    <div class="3">
                        <div class="4"></div>
                    </div>
                </div>
                <div class="5"></div>
            </div>
            "##;

            let tag = "div";
            let mut to = vec![];
            let open_pattern = format!("<{}", tag);
            let close_pattern = format!("</{}>", tag);
            let pattern = format!("{}|{}", regex::escape(&close_pattern), regex::escape(&open_pattern));
            let reader = &mut string.as_bytes();
            io::BufRead::read_until(reader, ">".as_bytes()[0], &mut to).unwrap();
            let (matched, _size) = read_until_pattern_async(reader, &pattern, &mut to)
                .await
                .unwrap();
            assert_eq!(str::from_utf8(&matched).unwrap(), "<div");

            let mut string = "$1131132$".as_bytes();
            let mut to = Vec::new();
            let pattern = "1132|115";
            let (matched, size) = read_until_pattern_async(&mut string, pattern, &mut to)
                .await
                .unwrap();
            assert_eq!(string, "$".as_bytes());
            assert_eq!(to, "$1131132".as_bytes());
            assert_eq!(size, 8);
            assert_eq!(str::from_utf8(&matched).unwrap(), "1132");

            let string = "<a />\
                <b></b>\
            </link>";
            let mut to = Vec::new();
            let pattern = "</link>|<link>";
            let (matched, size) = read_until_pattern_async(&mut string.as_bytes(), pattern, &mut to)
                .await
                .unwrap();
            assert_eq!(to, string.as_bytes());
            assert_eq!(std::str::from_utf8(&matched).unwrap(), "</link>");
            assert_eq!(size, string.len());

            let mut string = "1</link>".as_bytes();
            let mut to = Vec::new();
            let pattern = "123|890";
            let (matched, size) = read_until_pattern_async(&mut string, pattern, &mut to)
                .await
                .unwrap();
            assert_eq!(to, "1</link>".as_bytes());
            assert_eq!(size, 8);
            assert_eq!(matched, Vec::<u8>::new());
        });

        Ok(())
    }

}