rar_stream/

rar_files_package.rs

//! Multi-volume RAR archive parser.
//!
//! This module provides the main entry point for parsing RAR archives.
//! The [`RarFilesPackage`] struct handles single and multi-volume archives,
//! automatically stitching files that span multiple volumes.
//!
//! ## Quick Start
//!
//! ```rust,ignore
//! use rar_stream::{RarFilesPackage, ParseOptions, LocalFileMedia, FileMedia};
//! use std::sync::Arc;
//!
//! // Open a single RAR file
//! let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new("archive.rar")?);
//! let package = RarFilesPackage::new(vec![file]);
//!
//! // Parse with default options
//! let files = package.parse(ParseOptions::default()).await?;
//!
//! // Read file content
//! let content = files[0].read_to_end().await?;
//! ```
//!
//! ## Multi-Volume Archives
//!
//! For split archives, provide all volumes in order:
//!
//! ```rust,ignore
//! let volumes: Vec<Arc<dyn FileMedia>> = vec![
//!     Arc::new(LocalFileMedia::new("archive.part1.rar")?),
//!     Arc::new(LocalFileMedia::new("archive.part2.rar")?),
//!     Arc::new(LocalFileMedia::new("archive.part3.rar")?),
//! ];
//! let package = RarFilesPackage::new(volumes);
//! let files = package.parse(ParseOptions::default()).await?;
//! ```
//!
//! ## Filtering Files
//!
//! Use [`ParseOptions`] to filter or limit results:
//!
//! ```rust,ignore
//! let opts = ParseOptions {
//!     // Only include .txt files
//!     filter: Some(Box::new(|name, _index| name.ends_with(".txt"))),
//!     // Limit to first 10 matches
//!     max_files: Some(10),
//!     ..Default::default()
//! };
//! let txt_files = package.parse(opts).await?;
//! ```
//!
//! ## Encrypted Archives
//!
//! With the `crypto` feature enabled:
//!
//! ```rust,ignore
//! let opts = ParseOptions {
//!     password: Some("secret".to_string()),
//!     ..Default::default()
//! };
//! let files = package.parse(opts).await?;
//! ```
//!
//! ## Archive Information
//!
//! Get metadata about the archive without parsing all files:
//!
//! ```rust,ignore
//! let info = package.get_archive_info().await?;
//! println!("Format: {:?}", info.version);
//! println!("Solid: {}", info.is_solid);
//! println!("Has recovery: {}", info.has_recovery_record);
//! ```

use crate::error::{RarError, Result};
use crate::file_media::{FileMedia, ReadInterval};
use crate::inner_file::InnerFile;
use crate::parsing::{
    rar5::{Rar5ArchiveHeaderParser, Rar5EncryptionHeaderParser, Rar5FileHeaderParser},
    ArchiveHeaderParser, FileHeaderParser, MarkerHeaderParser, RarVersion, TerminatorHeaderParser,
};
use crate::rar_file_chunk::RarFileChunk;
use std::collections::HashMap;
use std::sync::Arc;

/// Default prefetch size for header parsing (32KB).
///
/// A single read of this size is done at the start of each volume. All header
/// parsing (marker, archive header, file headers) is served from this buffer.
/// Only if headers exceed this size is a follow-up read needed. Since RAR
/// headers are typically small (50-500 bytes each), 32KB covers hundreds of
/// files in a single I/O round trip.
const HEADER_PREFETCH_SIZE: u64 = 32 * 1024;

/// Buffered reader that minimizes I/O calls during header parsing.
///
/// Reads a large chunk from the underlying media up-front, then serves
/// sub-range reads from the buffer. Falls back to a direct read only when
/// the requested range extends beyond the prefetched data.
struct HeaderBuffer {
    data: Vec<u8>,
    /// Byte offset in the file where `data[0]` starts.
    file_offset: u64,
    /// Size of each prefetch read.
    prefetch_size: u64,
}

impl HeaderBuffer {
    /// Create a new header buffer by prefetching from the given offset.
    async fn new(media: &Arc<dyn FileMedia>, start: u64, prefetch_size: u64) -> Result<Self> {
        let end = (start + prefetch_size - 1).min(media.length().saturating_sub(1));
        if start > end {
            return Ok(Self {
                data: Vec::new(),
                file_offset: start,
                prefetch_size,
            });
        }
        let data = media.read_range(ReadInterval { start, end }).await?;
        Ok(Self {
            data,
            file_offset: start,
            prefetch_size,
        })
    }

    /// Read a range from the buffer, falling back to media if needed.
    async fn read(&mut self, media: &Arc<dyn FileMedia>, start: u64, end: u64) -> Result<Vec<u8>> {
        if end < start {
            return Ok(Vec::new());
        }
        let buf_start = self.file_offset;
        let buf_end = buf_start + self.data.len() as u64;

        // Fast path: fully within prefetched buffer
        if start >= buf_start && end < buf_end {
            let local_start = (start - buf_start) as usize;
            let local_end = (end - buf_start) as usize;
            return Ok(self.data[local_start..=local_end].to_vec());
        }

        // Slow path: need a fresh read from media (and refresh buffer from here)
        let prefetch_end = (start + self.prefetch_size - 1).min(media.length().saturating_sub(1));
        let read_end = end.max(prefetch_end);
        let data = media
            .read_range(ReadInterval {
                start,
                end: read_end,
            })
            .await?;
        let result = data[..(end - start + 1) as usize].to_vec();
        // Update buffer to serve future reads from this new position
        self.data = data;
        self.file_offset = start;
        Ok(result)
    }
}

/// Archive metadata returned by [`RarFilesPackage::get_archive_info`].
///
/// Contains information about the archive format, flags, and capabilities.
/// All fields are read from the archive header without decompressing any files.
///
/// # Example
///
/// ```rust,ignore
/// let info = package.get_archive_info().await?;
/// if info.has_encrypted_headers {
///     println!("Archive requires password to list files");
/// }
/// if info.is_solid {
///     println!("Solid archive: files must be extracted in order");
/// }
/// ```
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ArchiveInfo {
    /// Whether the archive has a recovery record for error correction.
    ///
    /// Recovery records allow repairing damaged archives using Reed-Solomon codes.
    pub has_recovery_record: bool,

    /// Whether the archive uses solid compression.
    ///
    /// In solid archives, files are compressed together as a single stream.
    /// This improves compression ratio but requires extracting files in order.
    pub is_solid: bool,

    /// Whether the archive is locked (cannot be modified).
    ///
    /// Locked archives cannot have files added, deleted, or modified.
    pub is_locked: bool,

    /// Whether the archive is split across multiple volumes.
    ///
    /// Multi-volume archives have files that span multiple `.rar`/`.rXX` files.
    pub is_multivolume: bool,

    /// Whether file headers are encrypted (requires password to list files).
    ///
    /// Only RAR5 archives created with `rar -hp` have encrypted headers.
    /// Without the password, even file names cannot be read.
    pub has_encrypted_headers: bool,

    /// RAR format version (RAR4 or RAR5).
    pub version: RarVersion,
}

/// Options for parsing RAR archives.
///
/// Use this struct to customize parsing behavior, including filtering,
/// limiting results, and providing passwords for encrypted archives.
///
/// # Example
///
/// ```rust,ignore
/// let opts = ParseOptions {
///     filter: Some(Box::new(|name, _| name.ends_with(".mp4"))),
///     max_files: Some(100),
///     #[cfg(feature = "crypto")]
///     password: Some("secret".to_string()),
/// };
/// ```
#[derive(Default)]
pub struct ParseOptions {
    /// Filter function: return `true` to include a file.
    ///
    /// The function receives the file name and its index (0-based).
    /// Only files where the filter returns `true` are included in results.
    pub filter: Option<Box<dyn Fn(&str, usize) -> bool + Send + Sync>>,

    /// Maximum number of files to return.
    ///
    /// Parsing stops after this many files are found. Useful for previewing
    /// large archives without parsing everything.
    pub max_files: Option<usize>,

    /// Prefetch buffer size for header parsing (default: 32KB).
    ///
    /// All headers in a volume are read from a single prefetched buffer
    /// of this size, minimizing I/O round-trips on slow media (e.g., HTTP
    /// range requests). Increase for archives with many files or large
    /// headers; decrease if memory is constrained.
    pub header_prefetch_size: Option<u64>,

    /// Password for encrypted archives.
    ///
    /// Required for archives with encrypted file data or headers.
    /// If the password is wrong, [`RarError::DecryptionFailed`] is returned.
    #[cfg(feature = "crypto")]
    pub password: Option<String>,
}

/// Encryption info for a file.
#[cfg(feature = "crypto")]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FileEncryptionInfo {
    /// RAR5 encryption (AES-256-CBC with PBKDF2)
    Rar5 {
        /// 16-byte salt for key derivation
        salt: [u8; 16],
        /// 16-byte initialization vector
        init_v: [u8; 16],
        /// Log2 of PBKDF2 iteration count
        lg2_count: u8,
    },
    /// RAR4 encryption (AES-256-CBC with custom SHA-1 KDF)
    Rar4 {
        /// 8-byte salt for key derivation
        salt: [u8; 8],
    },
}

/// Parsed file chunk with metadata.
struct ParsedChunk {
    name: String,
    chunk: RarFileChunk,
    continues_in_next: bool,
    unpacked_size: u64,
    method: u8,
    /// Dictionary size (log2), only for RAR5 compressed files
    dict_size_log: u8,
    rar_version: RarVersion,
    /// Whether this file is part of a solid archive
    is_solid: bool,
    /// Encryption info (if encrypted)
    #[cfg(feature = "crypto")]
    encryption: Option<FileEncryptionInfo>,
}

/// Multi-volume RAR archive parser.
pub struct RarFilesPackage {
    files: Vec<Arc<dyn FileMedia>>,
}

impl RarFilesPackage {
    /// Create a new [`RarFilesPackage`] from a list of volume files.
    ///
    /// Volumes are automatically sorted into the correct order
    /// (`.rar` first, then `.r00`, `.r01`, etc.).
    pub fn new(files: Vec<Arc<dyn FileMedia>>) -> Self {
        // Sort files by name to ensure correct order (.rar, .r00, .r01, ...)
        let mut files = files;
        files.sort_by(|a, b| Self::volume_order(a.name()).cmp(&Self::volume_order(b.name())));
        Self { files }
    }

    /// Get sort order for volume names.
    fn volume_order(name: &str) -> (u32, String) {
        let lower = name.to_lowercase();
        if lower.ends_with(".rar") {
            // Check for .partN.rar naming (e.g., archive.part1.rar, archive.part10.rar)
            if let Some(stem) = lower.strip_suffix(".rar") {
                if let Some(part_pos) = stem.rfind(".part") {
                    if let Ok(n) = stem[part_pos + 5..].parse::<u32>() {
                        return (n, lower);
                    }
                }
            }
            (0, lower) // .rar comes first (for old-style naming)
        } else {
            // Try to extract number from extension like .r00, .r01
            let ext = lower.rsplit('.').next().unwrap_or("");
            if ext.starts_with('r') && ext.len() == 3 {
                ext[1..]
                    .parse::<u32>()
                    .map(|n| (n + 1, lower.clone()))
                    .unwrap_or((1000, lower))
            } else {
                (1000, lower)
            }
        }
    }

    /// Get archive metadata from the first volume.
    ///
    /// This performs a single I/O read to fetch all headers needed for metadata.
    pub async fn get_archive_info(&self) -> Result<ArchiveInfo> {
        use crate::parsing::rar5::Rar5EncryptionHeaderParser;

        if self.files.is_empty() {
            return Err(RarError::NoFilesFound);
        }

        let rar_file = &self.files[0];
        let mut buf = HeaderBuffer::new(rar_file, 0, HEADER_PREFETCH_SIZE).await?;

        let marker_buf = buf.read(rar_file, 0, 7).await?;
        let marker = MarkerHeaderParser::parse(&marker_buf)?;

        match marker.version {
            RarVersion::Rar4 => {
                let archive_buf = buf
                    .read(
                        rar_file,
                        marker.size as u64,
                        marker.size as u64 + ArchiveHeaderParser::HEADER_SIZE as u64 - 1,
                    )
                    .await?;
                let archive = ArchiveHeaderParser::parse(&archive_buf)?;

                Ok(ArchiveInfo {
                    has_recovery_record: archive.has_recovery,
                    is_solid: archive.has_solid_attributes,
                    is_locked: archive.is_locked,
                    is_multivolume: archive.has_volume_attributes,
                    has_encrypted_headers: archive.is_block_encoded,
                    version: RarVersion::Rar4,
                })
            }
            RarVersion::Rar5 => {
                let header_buf = buf
                    .read(
                        rar_file,
                        marker.size as u64,
                        (marker.size as u64 + 255).min(rar_file.length() - 1),
                    )
                    .await?;

                let has_encrypted_headers =
                    Rar5EncryptionHeaderParser::is_encryption_header(&header_buf);

                if has_encrypted_headers {
                    Ok(ArchiveInfo {
                        has_encrypted_headers: true,
                        version: RarVersion::Rar5,
                        ..Default::default()
                    })
                } else {
                    let (archive, _) = Rar5ArchiveHeaderParser::parse(&header_buf)?;

                    Ok(ArchiveInfo {
                        has_recovery_record: archive.archive_flags.has_recovery_record,
                        is_solid: archive.archive_flags.is_solid,
                        is_locked: archive.archive_flags.is_locked,
                        is_multivolume: archive.archive_flags.is_volume,
                        has_encrypted_headers: false,
                        version: RarVersion::Rar5,
                    })
                }
            }
        }
    }

    /// Parse a single RAR file and extract file chunks.
    ///
    /// Prefetches a 32KB buffer to minimize I/O round-trips. All header
    /// parsing is served from this buffer; a follow-up read only happens
    /// if headers exceed 32KB.
    async fn parse_file(
        &self,
        rar_file: &Arc<dyn FileMedia>,
        opts: &ParseOptions,
    ) -> Result<Vec<ParsedChunk>> {
        // Prefetch headers with a single I/O read — this is the key
        // optimization for slow/remote media (HTTP range requests, etc.)
        let prefetch = opts.header_prefetch_size.unwrap_or(HEADER_PREFETCH_SIZE);
        let mut buf = HeaderBuffer::new(rar_file, 0, prefetch).await?;

        let marker_buf = buf.read(rar_file, 0, 7).await?;
        let marker = MarkerHeaderParser::parse(&marker_buf)?;

        match marker.version {
            RarVersion::Rar4 => {
                self.parse_rar4_file(rar_file, opts, marker.size as u64, &mut buf)
                    .await
            }
            RarVersion::Rar5 => self.parse_rar5_file(rar_file, opts, &mut buf).await,
        }
    }

    /// Parse a RAR4 format file.
    async fn parse_rar4_file(
        &self,
        rar_file: &Arc<dyn FileMedia>,
        opts: &ParseOptions,
        marker_size: u64,
        buf: &mut HeaderBuffer,
    ) -> Result<Vec<ParsedChunk>> {
        let mut chunks = Vec::new();
        let mut offset = marker_size;

        // Parse archive header
        let archive_buf = buf
            .read(
                rar_file,
                offset,
                offset + ArchiveHeaderParser::HEADER_SIZE as u64 - 1,
            )
            .await?;
        let archive = ArchiveHeaderParser::parse(&archive_buf)?;
        let is_solid = archive.has_solid_attributes;
        offset += archive.size as u64;

        let mut file_count = 0usize;
        let mut retrieved_count = 0usize;
        let terminator_size = TerminatorHeaderParser::HEADER_SIZE as u64;

        // Parse file headers
        while offset < rar_file.length().saturating_sub(terminator_size) {
            // Read enough bytes for header (but not more than available)
            let bytes_available = rar_file.length().saturating_sub(offset);
            let read_size = (FileHeaderParser::HEADER_SIZE as u64).min(bytes_available);

            if read_size < 32 {
                // Not enough for minimum header
                break;
            }

            let header_buf = buf.read(rar_file, offset, offset + read_size - 1).await?;

            let file_header = match FileHeaderParser::parse(&header_buf) {
                Ok(h) => h,
                Err(_) => break,
            };

            // Check if this is a file header (type 0x74 = 116)
            if file_header.header_type != 0x74 {
                break;
            }

            // Check encryption - with crypto feature, we can handle encrypted files
            #[cfg(not(feature = "crypto"))]
            if file_header.is_encrypted {
                return Err(RarError::EncryptedNotSupported);
            }

            let data_start = offset
                .checked_add(file_header.head_size as u64)
                .ok_or_else(|| RarError::InvalidOffset {
                    offset,
                    length: rar_file.length(),
                })?;
            let data_end = if file_header.packed_size > 0 {
                data_start
                    .checked_add(file_header.packed_size - 1)
                    .ok_or_else(|| RarError::InvalidOffset {
                        offset: data_start,
                        length: rar_file.length(),
                    })?
            } else {
                // Empty range: end < start so RarFileChunk::length() returns 0
                data_start.saturating_sub(1)
            };

            // Apply filter
            let include = match &opts.filter {
                Some(f) => f(&file_header.name, file_count),
                None => true,
            };

            if include {
                let chunk = RarFileChunk::new(rar_file.clone(), data_start, data_end);

                // Parse encryption info if present (RAR4)
                #[cfg(feature = "crypto")]
                let encryption = if file_header.is_encrypted {
                    file_header
                        .salt
                        .map(|salt| FileEncryptionInfo::Rar4 { salt })
                } else {
                    None
                };

                chunks.push(ParsedChunk {
                    name: file_header.name.clone(),
                    chunk,
                    continues_in_next: file_header.continues_in_next,
                    unpacked_size: file_header.unpacked_size,
                    method: file_header.method,
                    dict_size_log: 22, // RAR4 doesn't specify, use 4MB default
                    rar_version: RarVersion::Rar4,
                    is_solid,
                    #[cfg(feature = "crypto")]
                    encryption,
                });
                retrieved_count += 1;

                // Check max files limit
                if let Some(max) = opts.max_files {
                    if retrieved_count >= max {
                        break;
                    }
                }
            }

            offset = data_end + 1;
            file_count += 1;
        }

        Ok(chunks)
    }

    /// Parse an encrypted header.
    /// The format is: 16-byte IV + encrypted header data (padded to 16 bytes).
    #[cfg(feature = "crypto")]
    fn parse_encrypted_header<T, F>(
        &self,
        data: &[u8],
        crypto: &crate::crypto::Rar5Crypto,
        parser: F,
    ) -> Result<(T, usize)>
    where
        F: FnOnce(&[u8]) -> Result<(T, usize)>,
    {
        use crate::parsing::rar5::VintReader;

        if data.len() < 16 {
            return Err(RarError::InvalidHeader);
        }

        // First 16 bytes are the IV
        let mut iv = [0u8; 16];
        iv.copy_from_slice(&data[..16]);

        // Read enough encrypted data - we need to determine the header size
        // RAR5 encrypted headers have their size after CRC and before type
        // We'll decrypt a reasonable chunk and parse from there
        let encrypted_start = 16;

        // Read at least 256 bytes of encrypted data (should be enough for most headers)
        let available = data.len().saturating_sub(encrypted_start);
        if available < 16 {
            return Err(RarError::InvalidHeader);
        }

        // Round up to 16-byte boundary
        let decrypt_len = (available.min(512) / 16) * 16;
        if decrypt_len == 0 {
            return Err(RarError::InvalidHeader);
        }

        let mut decrypted = data[encrypted_start..encrypted_start + decrypt_len].to_vec();
        crypto
            .decrypt(&iv, &mut decrypted)
            .map_err(|e| RarError::DecryptionFailed(e.to_string()))?;

        // Parse the decrypted header
        let (result, _) = parser(&decrypted)?;

        // Calculate actual header size including CRC, size vint, and content
        // We need to read the header size from decrypted data
        let mut reader = VintReader::new(&decrypted[4..]); // Skip CRC32
        let header_size = reader.read().ok_or(RarError::InvalidHeader)?;
        let size_vint_len = reader.position();

        // Total encrypted size = CRC(4) + size_vint + header_content, rounded up to 16
        let plaintext_size = 4 + size_vint_len + header_size as usize;
        let encrypted_size = plaintext_size.div_ceil(16) * 16;

        // Total consumed = IV(16) + encrypted_size
        Ok((result, 16 + encrypted_size))
    }

    /// Parse a RAR5 format file.
    async fn parse_rar5_file(
        &self,
        rar_file: &Arc<dyn FileMedia>,
        opts: &ParseOptions,
        buf: &mut HeaderBuffer,
    ) -> Result<Vec<ParsedChunk>> {
        let mut chunks = Vec::new();
        let mut offset = 8u64; // RAR5 signature is 8 bytes

        // Read first header to check for encryption header
        let header_buf = buf
            .read(
                rar_file,
                offset,
                (offset + 256 - 1).min(rar_file.length() - 1),
            )
            .await?;

        // Check if headers are encrypted
        #[cfg(feature = "crypto")]
        let header_crypto: Option<crate::crypto::Rar5Crypto> =
            if Rar5EncryptionHeaderParser::is_encryption_header(&header_buf) {
                let (enc_header, consumed) = Rar5EncryptionHeaderParser::parse(&header_buf)?;
                offset += consumed as u64;

                // Need password to decrypt headers
                let password = opts.password.as_ref().ok_or(RarError::PasswordRequired)?;

                Some(crate::crypto::Rar5Crypto::derive_key(
                    password,
                    &enc_header.salt,
                    enc_header.lg2_count,
                ))
            } else {
                None
            };

        #[cfg(not(feature = "crypto"))]
        if Rar5EncryptionHeaderParser::is_encryption_header(&header_buf) {
            return Err(RarError::PasswordRequired);
        }

        // Read archive header (which may be encrypted)
        #[cfg(feature = "crypto")]
        let (archive_header, consumed) = if let Some(ref crypto) = header_crypto {
            // Read IV (16 bytes) + encrypted header
            let enc_buf = buf
                .read(
                    rar_file,
                    offset,
                    (offset + 512 - 1).min(rar_file.length() - 1),
                )
                .await?;

            self.parse_encrypted_header(&enc_buf, crypto, |data| {
                Rar5ArchiveHeaderParser::parse(data)
            })?
        } else {
            Rar5ArchiveHeaderParser::parse(&header_buf)?
        };

        #[cfg(not(feature = "crypto"))]
        let (archive_header, consumed) = Rar5ArchiveHeaderParser::parse(&header_buf)?;

        let is_solid = archive_header.archive_flags.is_solid;
        offset += consumed as u64;

        let mut file_count = 0usize;
        let mut retrieved_count = 0usize;

        // Parse file headers
        while offset < rar_file.length().saturating_sub(16) {
            // Read header data (variable size)
            let bytes_available = rar_file.length().saturating_sub(offset);
            let read_size = 512u64.min(bytes_available);

            if read_size < 16 {
                break;
            }

            let header_buf = buf.read(rar_file, offset, offset + read_size - 1).await?;

            // Try to parse as file header (may be encrypted)
            #[cfg(feature = "crypto")]
            let (file_header, header_consumed) = if let Some(ref crypto) = header_crypto {
                match self.parse_encrypted_header(&header_buf, crypto, |data| {
                    Rar5FileHeaderParser::parse(data)
                }) {
                    Ok(h) => h,
                    Err(_) => break,
                }
            } else {
                match Rar5FileHeaderParser::parse(&header_buf) {
                    Ok(h) => h,
                    Err(_) => break,
                }
            };

            #[cfg(not(feature = "crypto"))]
            let (file_header, header_consumed) = match Rar5FileHeaderParser::parse(&header_buf) {
                Ok(h) => h,
                Err(_) => break,
            };

            let data_start = offset.checked_add(header_consumed as u64).ok_or_else(|| {
                RarError::InvalidOffset {
                    offset,
                    length: rar_file.length(),
                }
            })?;
            let data_end = if file_header.packed_size > 0 {
                data_start
                    .checked_add(file_header.packed_size - 1)
                    .ok_or_else(|| RarError::InvalidOffset {
                        offset: data_start,
                        length: rar_file.length(),
                    })?
            } else {
                // Empty range: end < start so RarFileChunk::length() returns 0
                data_start.saturating_sub(1)
            };

            // Apply filter
            let include = match &opts.filter {
                Some(f) => f(&file_header.name, file_count),
                None => true,
            };

            if include {
                let chunk = RarFileChunk::new(rar_file.clone(), data_start, data_end);

                // Convert RAR5 method to RAR4-compatible format
                // RAR5 method 0 = stored, 1-5 = compression
                // Store the raw method, not converted to RAR4 format
                let method = file_header.compression.method;

                // Parse encryption info if present
                #[cfg(feature = "crypto")]
                let encryption = if file_header.is_encrypted() {
                    file_header.encryption_info().and_then(|data| {
                        crate::crypto::Rar5EncryptionInfo::parse(data)
                            .ok()
                            .map(|info| FileEncryptionInfo::Rar5 {
                                salt: info.salt,
                                init_v: info.init_v,
                                lg2_count: info.lg2_count,
                            })
                    })
                } else {
                    None
                };

                chunks.push(ParsedChunk {
                    name: file_header.name.clone(),
                    chunk,
                    continues_in_next: file_header.continues_in_next(),
                    unpacked_size: file_header.unpacked_size,
                    method,
                    dict_size_log: file_header.compression.dict_size_log,
                    rar_version: RarVersion::Rar5,
                    is_solid,
                    #[cfg(feature = "crypto")]
                    encryption,
                });
                retrieved_count += 1;

                if let Some(max) = opts.max_files {
                    if retrieved_count >= max {
                        break;
                    }
                }
            }

            offset = data_end + 1;
            file_count += 1;
        }

        Ok(chunks)
    }

    /// Parse all volumes and return inner files.
    pub async fn parse(&self, opts: ParseOptions) -> Result<Vec<InnerFile>> {
        if self.files.is_empty() {
            return Err(RarError::NoFilesFound);
        }

        let mut all_parsed: Vec<Vec<ParsedChunk>> = Vec::new();

        let mut i = 0;
        while i < self.files.len() {
            let file = &self.files[i];
            let chunks = self.parse_file(file, &opts).await?;

            if chunks.is_empty() {
                i += 1;
                continue;
            }

            // Get info from last chunk
            let continues = chunks.last().unwrap().continues_in_next;

            all_parsed.push(chunks);

            // Handle continuation - parse each continuation volume's headers
            // to find actual data offsets (they may differ between volumes)
            if continues {
                while i + 1 < self.files.len() {
                    i += 1;
                    let next_file = &self.files[i];

                    let cont_chunks = self.parse_file(next_file, &opts).await?;
                    if cont_chunks.is_empty() {
                        break;
                    }

                    let cont_continues = cont_chunks.last().unwrap().continues_in_next;
                    all_parsed.push(cont_chunks);

                    if !cont_continues {
                        break;
                    }
                }
            }

            i += 1;
        }

        // Flatten and group chunks by filename, keeping method info
        let all_chunks: Vec<ParsedChunk> = all_parsed.into_iter().flatten().collect();

        #[cfg(feature = "crypto")]
        type GroupValue = (
            Vec<RarFileChunk>,
            u8,
            u8, // dict_size_log
            u64,
            RarVersion,
            bool, // is_solid
            Option<FileEncryptionInfo>,
        );
        #[cfg(not(feature = "crypto"))]
        type GroupValue = (Vec<RarFileChunk>, u8, u8, u64, RarVersion, bool);

        let mut grouped: HashMap<String, GroupValue> = HashMap::new();
        for chunk in all_chunks {
            #[cfg(feature = "crypto")]
            let entry = grouped.entry(chunk.name).or_insert_with(|| {
                (
                    Vec::new(),
                    chunk.method,
                    chunk.dict_size_log,
                    chunk.unpacked_size,
                    chunk.rar_version,
                    chunk.is_solid,
                    chunk.encryption,
                )
            });
            #[cfg(not(feature = "crypto"))]
            let entry = grouped.entry(chunk.name).or_insert_with(|| {
                (
                    Vec::new(),
                    chunk.method,
                    chunk.dict_size_log,
                    chunk.unpacked_size,
                    chunk.rar_version,
                    chunk.is_solid,
                )
            });
            entry.0.push(chunk.chunk);
        }

        // Create InnerFile for each group
        #[cfg(feature = "crypto")]
        let password = opts.password.clone();

        let inner_files: Vec<InnerFile> = grouped
            .into_iter()
            .map(|(name, value)| {
                #[cfg(feature = "crypto")]
                {
                    let (
                        chunks,
                        method,
                        dict_size_log,
                        unpacked_size,
                        rar_version,
                        is_solid,
                        encryption,
                    ) = value;
                    let enc_info = encryption.map(|e| match e {
                        FileEncryptionInfo::Rar5 {
                            salt,
                            init_v,
                            lg2_count,
                        } => crate::inner_file::EncryptionInfo::Rar5 {
                            salt,
                            init_v,
                            lg2_count,
                        },
                        FileEncryptionInfo::Rar4 { salt } => {
                            crate::inner_file::EncryptionInfo::Rar4 { salt }
                        }
                    });
                    InnerFile::new_encrypted_with_solid_dict(
                        name,
                        chunks,
                        method,
                        dict_size_log,
                        unpacked_size,
                        rar_version,
                        enc_info,
                        password.clone(),
                        is_solid,
                    )
                }
                #[cfg(not(feature = "crypto"))]
                {
                    let (chunks, method, dict_size_log, unpacked_size, rar_version, is_solid) =
                        value;
                    InnerFile::new_with_solid_dict(
                        name,
                        chunks,
                        method,
                        dict_size_log,
                        unpacked_size,
                        rar_version,
                        is_solid,
                    )
                }
            })
            .collect();

        Ok(inner_files)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::file_media::{FileMedia, LocalFileMedia};

    #[test]
    fn test_volume_order_old_style() {
        // .rar comes first
        assert_eq!(RarFilesPackage::volume_order("archive.rar").0, 0);
        // .r00 = order 1, .r01 = order 2
        assert_eq!(RarFilesPackage::volume_order("archive.r00").0, 1);
        assert_eq!(RarFilesPackage::volume_order("archive.r01").0, 2);
        assert_eq!(RarFilesPackage::volume_order("archive.r99").0, 100);
    }

    #[test]
    fn test_volume_order_part_naming() {
        // .partN.rar naming should sort numerically
        let mut names = vec![
            "archive.part10.rar",
            "archive.part2.rar",
            "archive.part1.rar",
            "archive.part3.rar",
        ];
        names.sort_by_key(|n| RarFilesPackage::volume_order(n));
        assert_eq!(
            names,
            vec![
                "archive.part1.rar",
                "archive.part2.rar",
                "archive.part3.rar",
                "archive.part10.rar",
            ]
        );
    }

    #[test]
    fn test_volume_order_unknown() {
        // Unknown extensions get high order
        assert_eq!(RarFilesPackage::volume_order("archive.zip").0, 1000);
    }

    #[tokio::test]
    #[cfg(feature = "async")]
    async fn test_get_archive_info_rar5() {
        let file: Arc<dyn FileMedia> =
            Arc::new(LocalFileMedia::new("__fixtures__/rar5/test.rar").unwrap());
        let package = RarFilesPackage::new(vec![file]);

        let info = package.get_archive_info().await.unwrap();
        assert_eq!(info.version, RarVersion::Rar5);
        assert!(!info.is_multivolume);
    }

    #[tokio::test]
    #[cfg(feature = "async")]
    async fn test_get_archive_info_rar4() {
        let file: Arc<dyn FileMedia> =
            Arc::new(LocalFileMedia::new("__fixtures__/single/single.rar").unwrap());
        let package = RarFilesPackage::new(vec![file]);

        let info = package.get_archive_info().await.unwrap();
        assert_eq!(info.version, RarVersion::Rar4);
        assert!(!info.is_multivolume);
    }

    #[tokio::test]
    #[cfg(feature = "async")]
    async fn test_parse_rar5_stored() {
        // Test parsing a RAR5 stored file
        let file: Arc<dyn FileMedia> =
            Arc::new(LocalFileMedia::new("__fixtures__/rar5/test.rar").unwrap());
        let package = RarFilesPackage::new(vec![file]);

        let files = package.parse(ParseOptions::default()).await.unwrap();

        assert_eq!(files.len(), 1);
        assert_eq!(files[0].name, "test.txt");
    }

    #[tokio::test]
    #[cfg(feature = "async")]
    async fn test_parse_rar5_compressed() {
        // Test parsing a RAR5 compressed file
        let file: Arc<dyn FileMedia> =
            Arc::new(LocalFileMedia::new("__fixtures__/rar5/compressed.rar").unwrap());
        let package = RarFilesPackage::new(vec![file]);

        let files = package.parse(ParseOptions::default()).await.unwrap();

        assert_eq!(files.len(), 1);
        assert_eq!(files[0].name, "compress_test.txt");
        assert_eq!(files[0].length, 152); // Unpacked size

        // Try to read and decompress the file content
        // Note: RAR5 compressed decompression is still being debugged
        match files[0].read_to_end().await {
            Ok(content) => {
                eprintln!("Got {} bytes of output", content.len());
                eprintln!("First 32 bytes: {:02x?}", &content[..32.min(content.len())]);

                // Verify we got the full uncompressed content
                assert_eq!(
                    content.len(),
                    152,
                    "decompressed size should match unpacked size"
                );

                // Verify the content is valid text
                match std::str::from_utf8(&content) {
                    Ok(text) => {
                        assert!(
                            text.contains("This is a test file"),
                            "content should contain expected text"
                        );
                        assert!(
                            text.contains("hello hello"),
                            "content should contain repeated text"
                        );
                    }
                    Err(_) => {
                        // Decompression ran but output is wrong - still debugging
                        eprintln!(
                            "RAR5 decompression output is not valid UTF-8 (work in progress)"
                        );
                    }
                }
            }
            Err(e) => {
                // RAR5 decompression not yet fully implemented - parsing verified
                eprintln!("RAR5 decompression error: {:?}", e);
            }
        }
    }

    #[tokio::test]
    #[cfg(feature = "async")]
    async fn test_parse_rar5_multivolume() {
        // Test parsing a multi-volume RAR5 archive
        let fixture_dir = "__fixtures__/rar5-multivolume";

        // Collect all volume files
        let mut volume_paths: Vec<String> = std::fs::read_dir(fixture_dir)
            .unwrap()
            .filter_map(|e| e.ok())
            .map(|e| e.path())
            .filter(|p| p.extension().map_or(false, |ext| ext == "rar"))
            .map(|p| p.to_string_lossy().to_string())
            .collect();

        // Sort by name so volumes are in order
        volume_paths.sort();

        if volume_paths.is_empty() {
            // Skip test if fixtures don't exist
            eprintln!("Skipping test - no multi-volume fixtures found");
            return;
        }

        eprintln!("Found {} volumes: {:?}", volume_paths.len(), volume_paths);

        // Create file medias for each volume
        let files: Vec<Arc<dyn FileMedia>> = volume_paths
            .iter()
            .map(|p| Arc::new(LocalFileMedia::new(p).unwrap()) as Arc<dyn FileMedia>)
            .collect();

        let package = RarFilesPackage::new(files);

        let parsed = package.parse(ParseOptions::default()).await.unwrap();

        assert_eq!(parsed.len(), 1, "should have 1 inner file");
        assert_eq!(parsed[0].name, "testfile.txt");

        // The length might be slightly off due to volume header handling
        // but should be close to the original file size
        eprintln!("Parsed length: {}", parsed[0].length);

        // Try to read the file content (stored, so should work)
        let content = parsed[0].read_to_end().await.unwrap();
        eprintln!("Read content length: {}", content.len());

        // Verify the content is valid and contains expected text
        let text = std::str::from_utf8(&content).expect("should be valid UTF-8");
        assert!(text.contains("Line 1:"), "should contain first line");
        assert!(text.contains("Line 100:"), "should contain last line");

        // Verify we got approximately the right size (allow for header overhead)
        assert!(content.len() >= 11000, "should have at least 11000 bytes");
    }

    #[tokio::test]
    #[cfg(all(feature = "async", feature = "crypto"))]
    async fn test_parse_rar5_encrypted_stored() {
        // Test parsing and extracting an encrypted RAR5 file (stored, no compression)
        let fixture = "__fixtures__/encrypted/rar5-encrypted-stored.rar";

        if !std::path::Path::new(fixture).exists() {
            eprintln!("Skipping test - encrypted fixtures not found");
            return;
        }

        let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new(fixture).unwrap());
        let package = RarFilesPackage::new(vec![file]);

        let opts = ParseOptions {
            password: Some("testpass".to_string()),
            ..Default::default()
        };

        let parsed = package.parse(opts).await.unwrap();
        assert_eq!(parsed.len(), 1, "should have 1 inner file");

        let inner_file = &parsed[0];
        assert_eq!(inner_file.name, "testfile.txt");
        assert!(inner_file.is_encrypted());

        // Read the decrypted content
        let content = inner_file.read_decompressed().await.unwrap();
        let text = std::str::from_utf8(&content).expect("should be valid UTF-8");

        assert!(text.starts_with("Hello, encrypted world!"));
    }

    #[tokio::test]
    #[cfg(all(feature = "async", feature = "crypto"))]
    async fn test_parse_rar5_encrypted_no_password() {
        let fixture = "__fixtures__/encrypted/rar5-encrypted-stored.rar";

        if !std::path::Path::new(fixture).exists() {
            eprintln!("Skipping test - encrypted fixtures not found");
            return;
        }

        let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new(fixture).unwrap());
        let package = RarFilesPackage::new(vec![file]);

        // No password provided
        let parsed = package.parse(ParseOptions::default()).await.unwrap();
        assert_eq!(parsed.len(), 1, "should have 1 inner file");

        let inner_file = &parsed[0];
        assert!(inner_file.is_encrypted());

        // Reading should fail because no password was provided
        let result = inner_file.read_decompressed().await;
        assert!(result.is_err());
        match result {
            Err(crate::RarError::PasswordRequired) => {
                // Expected error
            }
            Err(e) => panic!("Expected PasswordRequired error, got: {:?}", e),
            Ok(_) => panic!("Expected error but got success"),
        }
    }

    #[tokio::test]
    #[cfg(all(feature = "async", feature = "crypto"))]
    async fn test_parse_rar5_encrypted_headers() {
        // Test parsing an archive with encrypted headers (created with rar -hp)
        let fixture = "__fixtures__/encrypted/rar5-encrypted-headers.rar";

        if !std::path::Path::new(fixture).exists() {
            eprintln!("Skipping test - encrypted headers fixture not found");
            return;
        }

        let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new(fixture).unwrap());
        let package = RarFilesPackage::new(vec![file]);

        // First check archive info - should show encrypted headers
        let info = package.get_archive_info().await.unwrap();
        assert!(info.has_encrypted_headers, "should have encrypted headers");
        assert_eq!(info.version, RarVersion::Rar5);

        // Parsing without password should fail
        let result = package.parse(ParseOptions::default()).await;
        assert!(
            matches!(result, Err(RarError::PasswordRequired)),
            "should require password for encrypted headers, got {:?}",
            result
        );

        // Parsing with password should succeed
        let opts = ParseOptions {
            password: Some("testpass".to_string()),
            ..Default::default()
        };

        let parsed = package.parse(opts).await.unwrap();
        assert_eq!(parsed.len(), 1, "should have 1 inner file");
        assert_eq!(parsed[0].name, "testfile.txt");

        // File content is also encrypted, so read should work
        let content = parsed[0].read_decompressed().await.unwrap();
        let text = std::str::from_utf8(&content).expect("should be valid UTF-8");
        assert!(
            text.starts_with("Hello, encrypted world!"),
            "content was: {:?}",
            text
        );
    }

    #[tokio::test]
    #[cfg(all(feature = "async", feature = "crypto"))]
    async fn test_get_archive_info_encrypted_headers() {
        // Test that get_archive_info detects encrypted headers
        let fixture = "__fixtures__/encrypted/rar5-encrypted-headers.rar";

        if !std::path::Path::new(fixture).exists() {
            eprintln!("Skipping test - encrypted headers fixture not found");
            return;
        }

        let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new(fixture).unwrap());
        let package = RarFilesPackage::new(vec![file]);

        let info = package.get_archive_info().await.unwrap();
        assert!(info.has_encrypted_headers);
        assert_eq!(info.version, RarVersion::Rar5);
        // Other flags can't be read when headers are encrypted
    }

    #[tokio::test]
    #[cfg(all(feature = "async", feature = "crypto"))]
    async fn test_parse_rar4_encrypted_stored() {
        // Test parsing and extracting an encrypted RAR4 file (stored, no compression)
        let fixture = "__fixtures__/encrypted/rar4-encrypted-stored.rar";

        if !std::path::Path::new(fixture).exists() {
            eprintln!("Skipping test - RAR4 encrypted fixtures not found");
            return;
        }

        let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new(fixture).unwrap());
        let package = RarFilesPackage::new(vec![file]);

        // Check archive info
        let info = package.get_archive_info().await.unwrap();
        assert_eq!(info.version, RarVersion::Rar4);

        let opts = ParseOptions {
            password: Some("testpass".to_string()),
            ..Default::default()
        };

        let parsed = package.parse(opts).await.unwrap();
        assert_eq!(parsed.len(), 1, "should have 1 inner file");

        let inner_file = &parsed[0];
        assert_eq!(inner_file.name, "testfile.txt");
        assert!(inner_file.is_encrypted());

        // Read the decrypted content
        let content = inner_file.read_decompressed().await.unwrap();
        let text = std::str::from_utf8(&content).expect("should be valid UTF-8");

        assert!(
            text.starts_with("Hello, encrypted world!"),
            "content was: {:?}",
            text
        );
    }

    #[tokio::test]
    #[cfg(all(feature = "async", feature = "crypto"))]
    async fn test_parse_rar4_encrypted_compressed() {
        // Test parsing and extracting an encrypted RAR4 file (compressed)
        let fixture = "__fixtures__/encrypted/rar4-encrypted.rar";

        if !std::path::Path::new(fixture).exists() {
            eprintln!("Skipping test - RAR4 encrypted fixtures not found");
            return;
        }

        let file: Arc<dyn FileMedia> = Arc::new(LocalFileMedia::new(fixture).unwrap());
        let package = RarFilesPackage::new(vec![file]);

        // Check archive info
        let info = package.get_archive_info().await.unwrap();
        assert_eq!(info.version, RarVersion::Rar4);

        let opts = ParseOptions {
            password: Some("testpass".to_string()),
            ..Default::default()
        };

        let parsed = package.parse(opts).await.unwrap();
        assert_eq!(parsed.len(), 1, "should have 1 inner file");

        let inner_file = &parsed[0];
        assert_eq!(inner_file.name, "testfile.txt");
        assert!(inner_file.is_encrypted());

        // Read the decrypted content
        let content = inner_file.read_decompressed().await.unwrap();
        let text = std::str::from_utf8(&content).expect("should be valid UTF-8");

        assert!(
            text.starts_with("Hello, encrypted world!"),
            "content was: {:?}",
            text
        );
    }
}