mdf4-rs 0.6.0

//! MDF File Indexing System
//!
//! This module provides a lightweight indexing system for MDF4 files, enabling
//! efficient random access to channel data without loading the entire file into
//! memory. Indexes can be serialized to JSON for caching and reuse.
//!
//! # Overview
//!
//! The MDF index system addresses a key challenge with large measurement files:
//! reading specific channel data efficiently. Instead of parsing the entire file
//! structure each time, you can:
//!
//! 1. **Build an index** that captures channel metadata and data locations
//! 2. **Save the index** to disk for reuse across sessions
//! 3. **Read channel data** by seeking directly to the relevant byte ranges
//!
//! This approach is particularly valuable for:
//! - Large files (hundreds of MB to GB)
//! - Remote files accessed via HTTP range requests
//! - Applications that need to read specific channels repeatedly
//!
//! # Index Contents
//!
//! An [`MdfIndex`] contains:
//! - Channel group metadata (names, record sizes, record counts)
//! - Channel metadata (names, data types, byte offsets, conversions)
//! - Data block locations (file offsets and sizes)
//!
//! # Performance Comparison
//!
//! | Operation | Full Parse | With Index |
//! |-----------|-----------|------------|
//! | Open 1GB file | 5-10 sec | <100ms |
//! | Read 1 channel | Full parse | ~50ms |
//! | Second channel | Full parse | ~50ms |
//!
//! # Example: Building and Using an Index
//!
//! ```no_run
//! use mdf4_rs::{MdfIndex, FileRangeReader, Result};
//!
//! fn read_efficiently() -> Result<()> {
//!     // Option 1: Create index with streaming (minimal memory)
//!     let index = MdfIndex::from_file_streaming("large_file.mf4")?;
//!
//!     // Save for later use (requires serde_json feature)
//!     index.save_to_file("large_file.index")?;
//!
//!     // Option 2: Load pre-built index (instant)
//!     let index = MdfIndex::load_from_file("large_file.index")?;
//!
//!     // Read only the channel you need
//!     let mut reader = FileRangeReader::new("large_file.mf4")?;
//!     let values = index.read_channel_values_by_name("Temperature", &mut reader)?;
//!
//!     Ok(())
//! }
//! ```
//!
//! # Reader Types
//!
//! The index system supports multiple reader implementations:
//!
//! - [`FileRangeReader`]: Direct file access (simple, low memory)
//! - [`BufferedRangeReader`]: Buffered file access (better for sequential reads)
//! - Custom implementations: HTTP range requests, cloud storage, etc.
//!
//! # Feature Flags
//!
//! - `serde`: Enables index serialization/deserialization
//! - `serde_json`: Enables JSON file save/load methods

mod _http_range_reader_example;
mod buffered_range_reader;
mod byte_range_reader;
mod data_block_info;
mod file_range_reader;
mod indexed_channel;
mod indexed_channel_group;
pub use _http_range_reader_example::_HttpRangeReaderExample;
pub use buffered_range_reader::BufferedRangeReader;
pub use byte_range_reader::ByteRangeReader;
pub use data_block_info::DataBlockInfo;
pub use file_range_reader::FileRangeReader;
pub use indexed_channel::IndexedChannel;
pub use indexed_channel_group::IndexedChannelGroup;

#[cfg(feature = "compression")]
use crate::blocks::DzBlock;
use crate::{
    Error, MDF, Result,
    blocks::{
        BlockHeader, BlockParse, ChannelBlock, ChannelGroupBlock, ConversionBlock, ConversionType,
        DataGroupBlock, DataListBlock, DataType, HeaderBlock, HlBlock, IdentificationBlock,
        TextBlock, u64_to_usize, validate_buffer_size,
    },
    parsing::decoder::{DecodedValue, decode_channel_value_with_validity},
};
use std::collections::BTreeMap;

/// Complete index of an MDF file for efficient random access.
///
/// The index captures all structural information needed to read channel
/// data without parsing the entire MDF file. It can be serialized to JSON
/// for caching across sessions.
///
/// # Creating an Index
///
/// ```no_run
/// use mdf4_rs::MdfIndex;
///
/// // From file (loads entire structure into memory)
/// let index = MdfIndex::from_file("data.mf4")?;
///
/// // From file with streaming (minimal memory)
/// let index = MdfIndex::from_file_streaming("large_file.mf4")?;
/// # Ok::<(), mdf4_rs::Error>(())
/// ```
///
/// # Reading Channel Data
///
/// ```no_run
/// use mdf4_rs::{MdfIndex, FileRangeReader};
///
/// let index = MdfIndex::from_file_streaming("data.mf4")?;
/// let mut reader = FileRangeReader::new("data.mf4")?;
///
/// // By name (searches all groups)
/// let values = index.read_channel_values_by_name("Temperature", &mut reader)?;
///
/// // By index (faster, no search)
/// let values = index.read_channel_values(0, 1, &mut reader)?;
/// # Ok::<(), mdf4_rs::Error>(())
/// ```
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct MdfIndex {
    /// Original file size in bytes (for validation)
    pub file_size: u64,
    /// All channel groups in the file
    pub channel_groups: Vec<IndexedChannelGroup>,
}

impl MdfIndex {
    /// Create an index from an MDF file
    pub fn from_file(file_path: &str) -> Result<Self> {
        let mdf = MDF::from_file(file_path)?;
        let file_size = std::fs::metadata(file_path).map_err(Error::IOError)?.len();

        let mut indexed_groups = Vec::new();

        for group in mdf.channel_groups() {
            let mut indexed_channels = Vec::new();
            let mmap = group.mmap(); // Get memory mapped file data for resolving conversions

            // Index each channel in the group
            for channel in group.channels() {
                let block = channel.block();

                // Clone and resolve conversion dependencies if present
                let resolved_conversion = if let Some(mut conversion) = block.conversion.clone() {
                    // Resolve all dependencies for this conversion block
                    if let Err(e) = conversion.resolve_all_dependencies(mmap) {
                        eprintln!(
                            "Warning: Failed to resolve conversion dependencies for channel '{}': {}",
                            block.name.as_deref().unwrap_or("<unnamed>"),
                            e
                        );
                    }
                    Some(conversion)
                } else {
                    None
                };

                let indexed_channel = IndexedChannel {
                    name: channel.name()?,
                    unit: channel.unit()?,
                    data_type: block.data_type,
                    byte_offset: block.byte_offset,
                    bit_offset: block.bit_offset,
                    bit_count: block.bit_count,
                    channel_type: block.channel_type,
                    flags: block.flags,
                    pos_invalidation_bit: block.pos_invalidation_bit,
                    conversion: resolved_conversion,
                    vlsd_data_address: if block.channel_type == 1 && block.data_addr != 0 {
                        Some(block.data_addr)
                    } else {
                        None
                    },
                };
                indexed_channels.push(indexed_channel);
            }

            // Get data block information
            let data_blocks = Self::extract_data_blocks(&group)?;

            let indexed_group = IndexedChannelGroup {
                name: group.name()?,
                comment: group.comment()?,
                record_id_size: group.raw_data_group().block.record_id_size,
                record_size: group.raw_channel_group().block.record_size,
                invalidation_bytes: group.raw_channel_group().block.invalidation_size,
                record_count: group.raw_channel_group().block.cycle_count,
                channels: indexed_channels,
                data_blocks,
            };
            indexed_groups.push(indexed_group);
        }

        Ok(MdfIndex {
            file_size,
            channel_groups: indexed_groups,
        })
    }

    /// Create an index from a file using streaming reads (minimal memory usage).
    ///
    /// This method reads only the metadata blocks needed to build the index,
    /// without loading the entire file into memory. Ideal for large files.
    ///
    /// # Arguments
    /// * `file_path` - Path to the MDF file
    ///
    /// # Example
    /// ```no_run
    /// use mdf4_rs::MdfIndex;
    ///
    /// let index = MdfIndex::from_file_streaming("large_recording.mf4")?;
    /// # Ok::<(), mdf4_rs::Error>(())
    /// ```
    pub fn from_file_streaming(file_path: &str) -> Result<Self> {
        let file_size = std::fs::metadata(file_path).map_err(Error::IOError)?.len();
        let mut reader = BufferedRangeReader::new(file_path)?;
        Self::from_reader(&mut reader, file_size)
    }

    /// Create an index from any byte range reader.
    ///
    /// This is the most flexible method, allowing index creation from files,
    /// HTTP sources, or any other data source implementing `ByteRangeReader`.
    ///
    /// # Arguments
    /// * `reader` - Any implementation of `ByteRangeReader`
    /// * `file_size` - Total size of the file in bytes
    pub fn from_reader<R: ByteRangeReader<Error = Error>>(
        reader: &mut R,
        file_size: u64,
    ) -> Result<Self> {
        // Read and validate ID block (64 bytes at offset 0)
        let id_bytes = reader.read_range(0, 64)?;
        let _id_block = IdentificationBlock::from_bytes(&id_bytes)?;

        // Read HD block (104 bytes at offset 64)
        let hd_bytes = reader.read_range(64, 104)?;
        let header = HeaderBlock::from_bytes(&hd_bytes)?;

        let mut indexed_groups = Vec::new();

        // Follow the DG chain
        let mut dg_addr = header.first_dg_addr;
        while dg_addr != 0 {
            // Read DG block (64 bytes)
            let dg_bytes = reader.read_range(dg_addr, 64)?;
            let dg_block = DataGroupBlock::from_bytes(&dg_bytes)?;

            // Follow the CG chain within this DG
            let mut cg_addr = dg_block.first_cg_addr;
            while cg_addr != 0 {
                // Read CG block (104 bytes)
                let cg_bytes = reader.read_range(cg_addr, 104)?;
                let cg_block = ChannelGroupBlock::from_bytes(&cg_bytes)?;

                // Read CG name if present
                let cg_name = Self::read_text_block(reader, cg_block.acq_name_addr)?;
                let cg_comment = Self::read_text_block(reader, cg_block.comment_addr)?;

                // Follow the CN chain within this CG
                let mut indexed_channels = Vec::new();
                let mut cn_addr = cg_block.first_ch_addr;
                while cn_addr != 0 {
                    // Read CN block (160 bytes)
                    let cn_bytes = reader.read_range(cn_addr, 160)?;
                    let cn_block = ChannelBlock::from_bytes(&cn_bytes)?;

                    // Read channel name
                    let ch_name = Self::read_text_block(reader, cn_block.name_addr)?;

                    // Read unit
                    let ch_unit = Self::read_text_block(reader, cn_block.unit_addr)?;

                    // Read and resolve conversion block if present
                    let conversion =
                        Self::read_conversion_block_streaming(reader, cn_block.conversion_addr)?;

                    let indexed_channel = IndexedChannel {
                        name: ch_name,
                        unit: ch_unit,
                        data_type: cn_block.data_type,
                        byte_offset: cn_block.byte_offset,
                        bit_offset: cn_block.bit_offset,
                        bit_count: cn_block.bit_count,
                        channel_type: cn_block.channel_type,
                        flags: cn_block.flags,
                        pos_invalidation_bit: cn_block.pos_invalidation_bit,
                        conversion,
                        vlsd_data_address: if cn_block.channel_type == 1 && cn_block.data_addr != 0
                        {
                            Some(cn_block.data_addr)
                        } else {
                            None
                        },
                    };
                    indexed_channels.push(indexed_channel);

                    cn_addr = cn_block.next_ch_addr;
                }

                // Extract data block info for this CG
                let data_blocks =
                    Self::extract_data_blocks_streaming(reader, dg_block.data_block_addr)?;

                let indexed_group = IndexedChannelGroup {
                    name: cg_name,
                    comment: cg_comment,
                    record_id_size: dg_block.record_id_size,
                    record_size: cg_block.record_size,
                    invalidation_bytes: cg_block.invalidation_size,
                    record_count: cg_block.cycle_count,
                    channels: indexed_channels,
                    data_blocks,
                };
                indexed_groups.push(indexed_group);

                cg_addr = cg_block.next_cg_addr;
            }

            dg_addr = dg_block.next_dg_addr;
        }

        Ok(MdfIndex {
            file_size,
            channel_groups: indexed_groups,
        })
    }

    /// Read a text block at the given address, returning None if address is 0.
    fn read_text_block<R: ByteRangeReader<Error = Error>>(
        reader: &mut R,
        addr: u64,
    ) -> Result<Option<String>> {
        if addr == 0 {
            return Ok(None);
        }

        // First read the header to get block length (24 bytes)
        let header_bytes = reader.read_range(addr, 24)?;
        let header = BlockHeader::from_bytes(&header_bytes)?;

        // Now read the full block
        let block_bytes = reader.read_range(addr, header.length)?;
        let text_block = TextBlock::from_bytes(&block_bytes)?;

        Ok(Some(text_block.text))
    }

    /// Read and parse a conversion block at the given address.
    fn read_conversion_block_streaming<R: ByteRangeReader<Error = Error>>(
        reader: &mut R,
        addr: u64,
    ) -> Result<Option<ConversionBlock>> {
        if addr == 0 {
            return Ok(None);
        }

        // First read the header to get block length
        let header_bytes = reader.read_range(addr, 24)?;
        let header = BlockHeader::from_bytes(&header_bytes)?;

        // Read the full conversion block
        let block_bytes = reader.read_range(addr, header.length)?;
        let mut conv_block = ConversionBlock::from_bytes(&block_bytes)?;

        // Resolve references based on conversion type
        Self::resolve_conversion_refs(reader, &mut conv_block)?;

        Ok(Some(conv_block))
    }

    /// Resolve references in a conversion block based on its type.
    fn resolve_conversion_refs<R: ByteRangeReader<Error = Error>>(
        reader: &mut R,
        conv: &mut ConversionBlock,
    ) -> Result<()> {
        match conv.conversion_type {
            // Algebraic conversion - first cc_ref is formula text
            ConversionType::Algebraic => {
                if let Some(&formula_addr) = conv.refs.first() {
                    if formula_addr != 0 {
                        conv.formula = Self::read_text_block(reader, formula_addr)?;
                    }
                }
            }
            // Text-based conversions - resolve text references
            ConversionType::ValueToText
            | ConversionType::RangeToText
            | ConversionType::TextToValue
            | ConversionType::TextToText
            | ConversionType::BitfieldText => {
                let mut resolved = BTreeMap::new();
                for (idx, &ref_addr) in conv.refs.iter().enumerate() {
                    if ref_addr != 0 {
                        // Check if this is a text block or nested conversion
                        let header_bytes = reader.read_range(ref_addr, 24)?;
                        let header = BlockHeader::from_bytes(&header_bytes)?;

                        if header.id == "##TX" || header.id == "##MD" {
                            if let Ok(Some(text)) = Self::read_text_block(reader, ref_addr) {
                                resolved.insert(idx, text);
                            }
                        }
                        // Skip nested conversions for now - they're complex
                    }
                }
                if !resolved.is_empty() {
                    conv.resolved_texts = Some(resolved);
                }
            }
            // Linear and other numeric conversions don't need text resolution
            _ => {}
        }

        Ok(())
    }

    /// Extract data block information using streaming reads.
    fn extract_data_blocks_streaming<R: ByteRangeReader<Error = Error>>(
        reader: &mut R,
        data_addr: u64,
    ) -> Result<Vec<DataBlockInfo>> {
        let mut data_blocks = Vec::new();
        let mut current_addr = data_addr;

        while current_addr != 0 {
            // Read block header (24 bytes)
            let header_bytes = reader.read_range(current_addr, 24)?;
            let header = BlockHeader::from_bytes(&header_bytes)?;

            match header.id.as_str() {
                "##DT" | "##DV" => {
                    data_blocks.push(DataBlockInfo {
                        file_offset: current_addr,
                        size: header.length,
                        is_compressed: false,
                    });
                    current_addr = 0;
                }
                "##DZ" => {
                    data_blocks.push(DataBlockInfo {
                        file_offset: current_addr,
                        size: header.length,
                        is_compressed: true,
                    });
                    current_addr = 0;
                }
                "##DL" => {
                    // Read the full DL block
                    let dl_bytes = reader.read_range(current_addr, header.length)?;
                    let dl_block = DataListBlock::from_bytes(&dl_bytes)?;

                    // Process each fragment
                    for &fragment_addr in &dl_block.data_block_addrs {
                        if fragment_addr == 0 {
                            continue;
                        }
                        let mut frag_pos = fragment_addr;
                        loop {
                            let frag_hdr_bytes = reader.read_range(frag_pos, 24)?;
                            let frag_hdr = BlockHeader::from_bytes(&frag_hdr_bytes)?;
                            if frag_hdr.id.as_str() != "##HL" {
                                data_blocks.push(DataBlockInfo {
                                    file_offset: frag_pos,
                                    size: frag_hdr.length,
                                    is_compressed: frag_hdr.id == "##DZ",
                                });
                                break;
                            }
                            let hl_bytes = reader.read_range(frag_pos, frag_hdr.length)?;
                            frag_pos = HlBlock::next_block_addr(&hl_bytes)?;
                        }
                    }

                    current_addr = dl_block.next_dl_addr;
                }
                "##HL" => {
                    let hl_bytes = reader.read_range(current_addr, header.length)?;
                    current_addr = HlBlock::next_block_addr(&hl_bytes)?;
                }
                _ => {
                    // Unknown block type, stop
                    current_addr = 0;
                }
            }
        }

        Ok(data_blocks)
    }

    /// Extract data block information from a channel group
    fn extract_data_blocks(
        group: &crate::channel_group::ChannelGroup,
    ) -> Result<Vec<DataBlockInfo>> {
        let mut data_blocks = Vec::new();
        let raw_data_group = group.raw_data_group();
        let mmap = group.mmap();

        // Start at the group's primary data pointer
        let mut current_block_address = raw_data_group.block.data_block_addr;
        while current_block_address != 0 {
            let byte_offset = current_block_address as usize;

            // Read the block header
            let block_header = BlockHeader::from_bytes(&mmap[byte_offset..byte_offset + 24])?;

            match block_header.id.as_str() {
                "##DT" | "##DV" => {
                    // Single contiguous DataBlock
                    let data_block_info = DataBlockInfo {
                        file_offset: current_block_address,
                        size: block_header.length,
                        is_compressed: false,
                    };
                    data_blocks.push(data_block_info);
                    // No list to follow, we're done
                    current_block_address = 0;
                }
                "##DZ" => {
                    // Compressed data block
                    let data_block_info = DataBlockInfo {
                        file_offset: current_block_address,
                        size: block_header.length,
                        is_compressed: true,
                    };
                    data_blocks.push(data_block_info);
                    current_block_address = 0;
                }
                "##DL" => {
                    // Fragmented list of data blocks
                    let data_list_block = DataListBlock::from_bytes(&mmap[byte_offset..])?;

                    // Parse each fragment in this list
                    for &fragment_address in &data_list_block.data_block_addrs {
                        if fragment_address == 0 {
                            continue;
                        }
                        let (frag_addr, fragment_header) =
                            HlBlock::skip_hierarchy_blocks(mmap, fragment_address)?;

                        let is_compressed = fragment_header.id == "##DZ";
                        let data_block_info = DataBlockInfo {
                            file_offset: frag_addr,
                            size: fragment_header.length,
                            is_compressed,
                        };
                        data_blocks.push(data_block_info);
                    }

                    // Move to the next DLBLOCK in the chain (0 = end)
                    current_block_address = data_list_block.next_dl_addr;
                }
                "##HL" => {
                    let len = u64_to_usize(block_header.length, "##HL")?;
                    validate_buffer_size(&mmap[byte_offset..], len)?;
                    current_block_address =
                        HlBlock::next_block_addr(&mmap[byte_offset..byte_offset + len])?;
                }

                unexpected_id => {
                    return Err(Error::BlockIDError {
                        actual: unexpected_id.to_string(),
                        expected: "##DT / ##DV / ##DL / ##DZ / ##HL".to_string(),
                    });
                }
            }
        }

        Ok(data_blocks)
    }

    /// Save the index to a JSON file.
    ///
    /// Requires the `serde` and `serde_json` features.
    #[cfg(feature = "serde_json")]
    pub fn save_to_file(&self, index_path: &str) -> Result<()> {
        let json = serde_json::to_string_pretty(self).map_err(|e| {
            Error::BlockSerializationError(format!("JSON serialization failed: {}", e))
        })?;

        std::fs::write(index_path, json).map_err(Error::IOError)?;

        Ok(())
    }

    /// Load an index from a JSON file.
    ///
    /// Requires the `serde` and `serde_json` features.
    #[cfg(feature = "serde_json")]
    pub fn load_from_file(index_path: &str) -> Result<Self> {
        let json = std::fs::read_to_string(index_path).map_err(Error::IOError)?;

        let index: MdfIndex = serde_json::from_str(&json).map_err(|e| {
            Error::BlockSerializationError(format!("JSON deserialization failed: {}", e))
        })?;

        Ok(index)
    }

    /// Read channel values using the index and a byte range reader
    ///
    /// # Returns
    /// A vector of `Option<DecodedValue>` where:
    /// - `Some(value)` represents a valid decoded value
    /// - `None` represents an invalid value (invalidation bit set or decoding failed)
    pub fn read_channel_values<R: ByteRangeReader<Error = Error>>(
        &self,
        group_index: usize,
        channel_index: usize,
        reader: &mut R,
    ) -> Result<Vec<Option<DecodedValue>>> {
        let group = self
            .channel_groups
            .get(group_index)
            .ok_or_else(|| Error::BlockSerializationError("Invalid group index".to_string()))?;

        let channel = group
            .channels
            .get(channel_index)
            .ok_or_else(|| Error::BlockSerializationError("Invalid channel index".to_string()))?;

        // Handle VLSD channels differently
        if channel.channel_type == 1 && channel.vlsd_data_address.is_some() {
            return self.read_vlsd_channel_values(group, channel, reader);
        }

        // For regular channels, read from data blocks
        self.read_regular_channel_values(group, channel, reader)
    }

    /// Read values for a regular (non-VLSD) channel using byte range reader
    fn read_regular_channel_values<R: ByteRangeReader<Error = Error>>(
        &self,
        group: &IndexedChannelGroup,
        channel: &IndexedChannel,
        reader: &mut R,
    ) -> Result<Vec<Option<DecodedValue>>> {
        // Record structure: record_id + data_bytes + invalidation_bytes
        let record_size = group.record_id_size as usize
            + group.record_size as usize
            + group.invalidation_bytes as usize;
        let mut values = Vec::new();

        // Read from each data block
        for data_block in &group.data_blocks {
            // Get the block data, decompressing if needed
            let block_data: Vec<u8> = if data_block.is_compressed {
                #[cfg(feature = "compression")]
                {
                    // Read the full DZ block (header + compressed data)
                    let dz_bytes = reader.read_range(data_block.file_offset, data_block.size)?;
                    let dz_block = DzBlock::from_bytes(&dz_bytes)?;
                    dz_block.decompress()?
                }
                #[cfg(not(feature = "compression"))]
                {
                    return Err(Error::BlockSerializationError(
                        "Compressed blocks require the 'compression' feature".to_string(),
                    ));
                }
            } else {
                // Read the block data (skip 24-byte block header)
                reader.read_range(data_block.file_offset + 24, data_block.size - 24)?
            };

            // Process records in this block
            let record_count = block_data.len() / record_size;
            for i in 0..record_count {
                let record_start = i * record_size;
                let record_end = record_start + record_size;
                let record = &block_data[record_start..record_end];

                // Create a ChannelBlock for decoding
                let temp_channel_block = ChannelBlock {
                    header: BlockHeader {
                        id: "##CN".to_string(),
                        reserved: 0,
                        length: 160,
                        link_count: 8,
                    },
                    next_ch_addr: 0,
                    component_addr: 0,
                    name_addr: 0,
                    source_addr: 0,
                    conversion_addr: 0,
                    data_addr: 0,
                    unit_addr: 0,
                    comment_addr: 0,
                    channel_type: channel.channel_type,
                    sync_type: 0,
                    data_type: channel.data_type,
                    bit_offset: channel.bit_offset,
                    byte_offset: channel.byte_offset,
                    bit_count: channel.bit_count,
                    flags: channel.flags,
                    pos_invalidation_bit: channel.pos_invalidation_bit,
                    precision: 0,
                    reserved1: 0,
                    attachment_count: 0,
                    min_raw_value: 0.0,
                    max_raw_value: 0.0,
                    lower_limit: 0.0,
                    upper_limit: 0.0,
                    lower_ext_limit: 0.0,
                    upper_ext_limit: 0.0,
                    name: channel.name.clone(),
                    conversion: channel.conversion.clone(),
                };

                // Decode with validity checking
                if let Some(decoded) = decode_channel_value_with_validity(
                    record,
                    group.record_id_size as usize,
                    group.record_size,
                    &temp_channel_block,
                ) {
                    if decoded.is_valid {
                        // Apply conversion if present
                        let final_value = if let Some(conversion) = &channel.conversion {
                            conversion.apply_decoded(decoded.value, &[])?
                        } else {
                            decoded.value
                        };
                        values.push(Some(final_value));
                    } else {
                        // Invalid sample
                        values.push(None);
                    }
                } else {
                    // Decoding failed
                    values.push(None);
                }
            }
        }

        Ok(values)
    }

    /// Read values for a VLSD channel.
    ///
    /// VLSD channels store variable-length data in separate Signal Data (SD) blocks,
    /// rather than in the regular channel group data blocks. Each record has the format:
    /// `[u32 length][value bytes]`.
    fn read_vlsd_channel_values<R: ByteRangeReader<Error = Error>>(
        &self,
        _group: &IndexedChannelGroup,
        channel: &IndexedChannel,
        reader: &mut R,
    ) -> Result<Vec<Option<DecodedValue>>> {
        let vlsd_addr = channel.vlsd_data_address.ok_or_else(|| {
            Error::BlockSerializationError("VLSD channel has no data address".to_string())
        })?;

        if vlsd_addr == 0 {
            return Ok(Vec::new());
        }

        let mut values = Vec::new();

        // Collect all SD block addresses (may be direct SD or via DL chain)
        let sd_addresses = self.collect_vlsd_block_addresses(vlsd_addr, reader)?;

        // Process each SD block
        for sd_addr in sd_addresses {
            // Read the SD block header first to get its size
            let header_bytes = reader.read_range(sd_addr, 24)?;
            let header = BlockHeader::from_bytes(&header_bytes)?;

            if header.id != "##SD" {
                return Err(Error::BlockIDError {
                    actual: header.id,
                    expected: "##SD".to_string(),
                });
            }

            // Read the full SD block data (after header)
            let data_size = header.length.saturating_sub(24) as usize;
            if data_size == 0 {
                continue;
            }
            let sd_data = reader.read_range(sd_addr + 24, data_size as u64)?;

            // Parse VLSD records: [u32 length][value bytes]...
            let mut pos = 0;
            while pos + 4 <= sd_data.len() {
                // Read the length prefix (u32 little-endian)
                let len = u32::from_le_bytes([
                    sd_data[pos],
                    sd_data[pos + 1],
                    sd_data[pos + 2],
                    sd_data[pos + 3],
                ]) as usize;

                let value_start = pos + 4;
                let value_end = value_start + len;

                if value_end > sd_data.len() {
                    // Truncated record - stop parsing
                    break;
                }

                let record = &sd_data[value_start..value_end];

                // Decode the VLSD value
                if let Some(decoded) = self.decode_vlsd_value(record, channel) {
                    // Apply conversion if present
                    let final_value = if let Some(conversion) = &channel.conversion {
                        match conversion.apply_decoded(decoded.clone(), &[]) {
                            Ok(v) => v,
                            Err(_) => decoded, // Fall back to raw value on conversion error
                        }
                    } else {
                        decoded
                    };
                    values.push(Some(final_value));
                } else {
                    values.push(None);
                }

                pos = value_end;
            }
        }

        Ok(values)
    }

    /// Collect all SD block addresses from a VLSD data address.
    ///
    /// The address may point directly to an SD block, or to a DL (Data List) block
    /// that chains multiple SD blocks together.
    fn collect_vlsd_block_addresses<R: ByteRangeReader<Error = Error>>(
        &self,
        start_addr: u64,
        reader: &mut R,
    ) -> Result<Vec<u64>> {
        let mut addresses = Vec::new();
        let mut next_addr = start_addr;

        while next_addr != 0 {
            // Read block header to determine type
            let header_bytes = reader.read_range(next_addr, 24)?;
            let header = BlockHeader::from_bytes(&header_bytes)?;

            match header.id.as_str() {
                "##SD" => {
                    // Direct SD block
                    addresses.push(next_addr);
                    break;
                }
                "##DL" => {
                    // Data List block - read the full block to get addresses
                    let dl_size = header.length as usize;
                    let dl_bytes = reader.read_range(next_addr, dl_size as u64)?;
                    let dl_block = DataListBlock::from_bytes(&dl_bytes)?;

                    // Add all fragment addresses
                    for &frag_addr in &dl_block.data_block_addrs {
                        if frag_addr == 0 {
                            continue;
                        }
                        let mut pos = frag_addr;
                        loop {
                            let hd = reader.read_range(pos, 24)?;
                            let h = BlockHeader::from_bytes(&hd)?;
                            if h.id.as_str() != "##HL" {
                                addresses.push(pos);
                                break;
                            }
                            let hl_bytes = reader.read_range(pos, h.length)?;
                            pos = HlBlock::next_block_addr(&hl_bytes)?;
                        }
                    }

                    // Follow the chain
                    next_addr = dl_block.next_dl_addr;
                }
                "##HL" => {
                    let hl_bytes = reader.read_range(next_addr, header.length)?;
                    next_addr = HlBlock::next_block_addr(&hl_bytes)?;
                }
                other => {
                    return Err(Error::BlockIDError {
                        actual: other.to_string(),
                        expected: "##SD or ##DL or ##HL".to_string(),
                    });
                }
            }
        }

        Ok(addresses)
    }

    /// Decode a VLSD value from its raw bytes.
    fn decode_vlsd_value(&self, record: &[u8], channel: &IndexedChannel) -> Option<DecodedValue> {
        if record.is_empty() {
            return None;
        }

        // For VLSD, the entire record is the value payload
        match channel.data_type {
            DataType::StringLatin1 => {
                // Latin-1 to UTF-8 conversion
                let text: String = record.iter().map(|&b| b as char).collect();
                let trimmed = text.trim_end_matches('\0').to_string();
                Some(DecodedValue::String(trimmed))
            }
            DataType::StringUtf8 => {
                let text = String::from_utf8_lossy(record);
                let trimmed = text.trim_end_matches('\0').to_string();
                Some(DecodedValue::String(trimmed))
            }
            DataType::StringUtf16LE => {
                if record.len() >= 2 {
                    let u16_values: Vec<u16> = record
                        .chunks_exact(2)
                        .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
                        .collect();
                    let text = String::from_utf16_lossy(&u16_values);
                    let trimmed = text.trim_end_matches('\0').to_string();
                    Some(DecodedValue::String(trimmed))
                } else {
                    None
                }
            }
            DataType::StringUtf16BE => {
                if record.len() >= 2 {
                    let u16_values: Vec<u16> = record
                        .chunks_exact(2)
                        .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
                        .collect();
                    let text = String::from_utf16_lossy(&u16_values);
                    let trimmed = text.trim_end_matches('\0').to_string();
                    Some(DecodedValue::String(trimmed))
                } else {
                    None
                }
            }
            DataType::ByteArray | DataType::MimeSample | DataType::MimeStream => {
                Some(DecodedValue::ByteArray(record.to_vec()))
            }
            // For numeric types, interpret based on size
            DataType::UnsignedIntegerLE => match record.len() {
                1 => Some(DecodedValue::UnsignedInteger(record[0] as u64)),
                2 => Some(DecodedValue::UnsignedInteger(
                    u16::from_le_bytes([record[0], record[1]]) as u64,
                )),
                4 => Some(DecodedValue::UnsignedInteger(u32::from_le_bytes([
                    record[0], record[1], record[2], record[3],
                ]) as u64)),
                8 => Some(DecodedValue::UnsignedInteger(u64::from_le_bytes([
                    record[0], record[1], record[2], record[3], record[4], record[5], record[6],
                    record[7],
                ]))),
                _ => Some(DecodedValue::ByteArray(record.to_vec())),
            },
            DataType::SignedIntegerLE => match record.len() {
                1 => Some(DecodedValue::SignedInteger(record[0] as i8 as i64)),
                2 => Some(DecodedValue::SignedInteger(
                    i16::from_le_bytes([record[0], record[1]]) as i64,
                )),
                4 => Some(DecodedValue::SignedInteger(i32::from_le_bytes([
                    record[0], record[1], record[2], record[3],
                ]) as i64)),
                8 => Some(DecodedValue::SignedInteger(i64::from_le_bytes([
                    record[0], record[1], record[2], record[3], record[4], record[5], record[6],
                    record[7],
                ]))),
                _ => Some(DecodedValue::ByteArray(record.to_vec())),
            },
            DataType::FloatLE => match record.len() {
                4 => Some(DecodedValue::Float(f32::from_le_bytes([
                    record[0], record[1], record[2], record[3],
                ]) as f64)),
                8 => Some(DecodedValue::Float(f64::from_le_bytes([
                    record[0], record[1], record[2], record[3], record[4], record[5], record[6],
                    record[7],
                ]))),
                _ => Some(DecodedValue::ByteArray(record.to_vec())),
            },
            _ => {
                // For other types, return as byte array
                Some(DecodedValue::ByteArray(record.to_vec()))
            }
        }
    }

    /// Get channel information for a specific group and channel
    pub fn get_channel_info(
        &self,
        group_index: usize,
        channel_index: usize,
    ) -> Option<&IndexedChannel> {
        self.channel_groups
            .get(group_index)?
            .channels
            .get(channel_index)
    }

    /// List all channel groups with their basic information
    pub fn list_channel_groups(&self) -> Vec<(usize, &str, usize)> {
        self.channel_groups
            .iter()
            .enumerate()
            .map(|(i, group)| {
                (
                    i,
                    group.name.as_deref().unwrap_or("<unnamed>"),
                    group.channels.len(),
                )
            })
            .collect()
    }

    /// List all channels in a specific group
    pub fn list_channels(&self, group_index: usize) -> Option<Vec<(usize, &str, &DataType)>> {
        let group = self.channel_groups.get(group_index)?;
        Some(
            group
                .channels
                .iter()
                .enumerate()
                .map(|(i, ch)| (i, ch.name.as_deref().unwrap_or("<unnamed>"), &ch.data_type))
                .collect(),
        )
    }

    /// Get the exact byte ranges needed to read all data for a specific channel
    ///
    /// Returns a vector of (file_offset, length) tuples representing the byte ranges
    /// that need to be read from the file to get all data for the specified channel.
    ///
    /// # Arguments
    /// * `group_index` - Index of the channel group
    /// * `channel_index` - Index of the channel within the group
    ///
    /// # Returns
    /// * `Ok(Vec<(u64, u64)>)` - Vector of (offset, length) byte ranges
    /// * `Err(MdfError)` - If indices are invalid or channel type not supported
    pub fn get_channel_byte_ranges(
        &self,
        group_index: usize,
        channel_index: usize,
    ) -> Result<Vec<(u64, u64)>> {
        let group = self
            .channel_groups
            .get(group_index)
            .ok_or_else(|| Error::BlockSerializationError("Invalid group index".to_string()))?;

        let channel = group
            .channels
            .get(channel_index)
            .ok_or_else(|| Error::BlockSerializationError("Invalid channel index".to_string()))?;

        // Handle VLSD channels differently
        if channel.channel_type == 1 && channel.vlsd_data_address.is_some() {
            return Err(Error::BlockSerializationError(
                "VLSD channels not yet supported for byte range calculation".to_string(),
            ));
        }

        // For regular channels, calculate byte ranges from data blocks
        self.calculate_regular_channel_byte_ranges(group, channel)
    }

    /// Get the exact byte ranges for a specific record range of a channel
    ///
    /// This is useful when you only want to read a subset of records rather than all data.
    ///
    /// # Arguments
    /// * `group_index` - Index of the channel group
    /// * `channel_index` - Index of the channel within the group
    /// * `start_record` - Starting record index (0-based)
    /// * `record_count` - Number of records to read
    ///
    /// # Returns
    /// * `Ok(Vec<(u64, u64)>)` - Vector of (offset, length) byte ranges
    /// * `Err(MdfError)` - If indices are invalid, range is out of bounds, or channel type not supported
    pub fn get_channel_byte_ranges_for_records(
        &self,
        group_index: usize,
        channel_index: usize,
        start_record: u64,
        record_count: u64,
    ) -> Result<Vec<(u64, u64)>> {
        let group = self
            .channel_groups
            .get(group_index)
            .ok_or_else(|| Error::BlockSerializationError("Invalid group index".to_string()))?;

        let channel = group
            .channels
            .get(channel_index)
            .ok_or_else(|| Error::BlockSerializationError("Invalid channel index".to_string()))?;

        // Validate record range
        if start_record + record_count > group.record_count {
            return Err(Error::BlockSerializationError(format!(
                "Record range {}-{} exceeds total records {}",
                start_record,
                start_record + record_count - 1,
                group.record_count
            )));
        }

        // Handle VLSD channels differently
        if channel.channel_type == 1 && channel.vlsd_data_address.is_some() {
            return Err(Error::BlockSerializationError(
                "VLSD channels not yet supported for byte range calculation".to_string(),
            ));
        }

        self.calculate_channel_byte_ranges_for_records(group, channel, start_record, record_count)
    }

    /// Calculate byte ranges for a regular (non-VLSD) channel for all records
    fn calculate_regular_channel_byte_ranges(
        &self,
        group: &IndexedChannelGroup,
        channel: &IndexedChannel,
    ) -> Result<Vec<(u64, u64)>> {
        self.calculate_channel_byte_ranges_for_records(group, channel, 0, group.record_count)
    }

    /// Calculate byte ranges for a regular channel for a specific record range
    fn calculate_channel_byte_ranges_for_records(
        &self,
        group: &IndexedChannelGroup,
        channel: &IndexedChannel,
        start_record: u64,
        record_count: u64,
    ) -> Result<Vec<(u64, u64)>> {
        // Record structure: record_id + data_bytes + invalidation_bytes
        let record_size = group.record_id_size as usize
            + group.record_size as usize
            + group.invalidation_bytes as usize;
        let channel_offset_in_record = group.record_id_size as usize + channel.byte_offset as usize;

        // Calculate how many bytes this channel needs per record
        let channel_bytes_per_record = if matches!(
            channel.data_type,
            DataType::StringLatin1
                | DataType::StringUtf8
                | DataType::StringUtf16LE
                | DataType::StringUtf16BE
                | DataType::ByteArray
                | DataType::MimeSample
                | DataType::MimeStream
        ) {
            channel.bit_count as usize / 8
        } else {
            (channel.bit_offset as usize + channel.bit_count as usize)
                .div_ceil(8)
                .max(1)
        };

        let mut byte_ranges = Vec::new();
        let mut records_processed = 0u64;

        for data_block in &group.data_blocks {
            if data_block.is_compressed {
                // Compressed blocks cannot be accessed via byte ranges because the
                // data layout changes after decompression. Use read_channel_values()
                // instead, which handles decompression transparently.
                return Err(Error::BlockSerializationError(
                    "Compressed blocks cannot be accessed via byte ranges. \
                     Use read_channel_values() instead."
                        .to_string(),
                ));
            }

            let block_data_start = data_block.file_offset + 24; // Skip block header
            let block_data_size = data_block.size - 24;
            let records_in_block = block_data_size / record_size as u64;

            // Determine which records from this block we need
            let block_start_record = records_processed;
            let block_end_record = records_processed + records_in_block;

            let need_start = start_record.max(block_start_record);
            let need_end = (start_record + record_count).min(block_end_record);

            if need_start < need_end {
                // We need some records from this block
                let first_record_in_block = need_start - block_start_record;
                let last_record_in_block = need_end - block_start_record - 1;

                // Calculate byte range for the channel data in these records
                let first_channel_byte = block_data_start
                    + first_record_in_block * record_size as u64
                    + channel_offset_in_record as u64;

                let last_channel_byte = block_data_start
                    + last_record_in_block * record_size as u64
                    + channel_offset_in_record as u64
                    + channel_bytes_per_record as u64
                    - 1;

                let range_length = last_channel_byte - first_channel_byte + 1;
                byte_ranges.push((first_channel_byte, range_length));
            }

            records_processed = block_end_record;

            // Early exit if we've processed all needed records
            if records_processed >= start_record + record_count {
                break;
            }
        }

        Ok(byte_ranges)
    }

    /// Get a summary of byte ranges for a channel (total bytes, number of ranges)
    ///
    /// This is useful for understanding the I/O pattern before actually reading.
    ///
    /// # Returns
    /// * `(total_bytes, number_of_ranges)` - Total bytes to read and number of separate ranges
    pub fn get_channel_byte_summary(
        &self,
        group_index: usize,
        channel_index: usize,
    ) -> Result<(u64, usize)> {
        let ranges = self.get_channel_byte_ranges(group_index, channel_index)?;
        let total_bytes: u64 = ranges.iter().map(|(_, len)| len).sum();
        Ok((total_bytes, ranges.len()))
    }

    /// Find a channel group index by name
    ///
    /// # Arguments
    /// * `group_name` - Name of the channel group to find
    ///
    /// # Returns
    /// * `Some(group_index)` if found
    /// * `None` if not found
    pub fn find_channel_group_by_name(&self, group_name: &str) -> Option<usize> {
        self.channel_groups
            .iter()
            .enumerate()
            .find(|(_, group)| group.name.as_deref() == Some(group_name))
            .map(|(index, _)| index)
    }

    /// Find a channel index by name within a specific group
    ///
    /// # Arguments
    /// * `group_index` - Index of the channel group to search in
    /// * `channel_name` - Name of the channel to find
    ///
    /// # Returns
    /// * `Some(channel_index)` if found
    /// * `None` if group doesn't exist or channel not found
    pub fn find_channel_by_name(&self, group_index: usize, channel_name: &str) -> Option<usize> {
        let group = self.channel_groups.get(group_index)?;

        group
            .channels
            .iter()
            .enumerate()
            .find(|(_, channel)| channel.name.as_deref() == Some(channel_name))
            .map(|(index, _)| index)
    }

    /// Find a channel by name across all groups
    ///
    /// # Arguments
    /// * `channel_name` - Name of the channel to find
    ///
    /// # Returns
    /// * `Some((group_index, channel_index))` if found
    /// * `None` if not found
    pub fn find_channel_by_name_global(&self, channel_name: &str) -> Option<(usize, usize)> {
        for (group_index, group) in self.channel_groups.iter().enumerate() {
            for (channel_index, channel) in group.channels.iter().enumerate() {
                if channel.name.as_deref() == Some(channel_name) {
                    return Some((group_index, channel_index));
                }
            }
        }
        None
    }

    /// Find all channels with a given name across all groups
    ///
    /// This is useful when the same channel name appears in multiple groups.
    ///
    /// # Arguments
    /// * `channel_name` - Name of the channels to find
    ///
    /// # Returns
    /// * `Vec<(group_index, channel_index)>` - All matching channels
    pub fn find_all_channels_by_name(&self, channel_name: &str) -> Vec<(usize, usize)> {
        let mut matches = Vec::new();

        for (group_index, group) in self.channel_groups.iter().enumerate() {
            for (channel_index, channel) in group.channels.iter().enumerate() {
                if channel.name.as_deref() == Some(channel_name) {
                    matches.push((group_index, channel_index));
                }
            }
        }

        matches
    }

    /// Read channel values by name using a byte range reader
    ///
    /// Convenience method that finds the channel by name and reads its values.
    /// If multiple channels have the same name, uses the first one found.
    ///
    /// # Arguments
    /// * `channel_name` - Name of the channel to read
    /// * `reader` - Byte range reader implementation
    ///
    /// # Returns
    /// * `Ok(Vec<Option<DecodedValue>>)` - Channel values (None for invalid samples)
    /// * `Err(MdfError)` - If channel not found or reading fails
    pub fn read_channel_values_by_name<R: ByteRangeReader<Error = Error>>(
        &self,
        channel_name: &str,
        reader: &mut R,
    ) -> Result<Vec<Option<DecodedValue>>> {
        let (group_index, channel_index) = self
            .find_channel_by_name_global(channel_name)
            .ok_or_else(|| {
                Error::BlockSerializationError(format!("Channel '{}' not found", channel_name))
            })?;

        self.read_channel_values(group_index, channel_index, reader)
    }

    /// Get byte ranges for a channel by name
    ///
    /// # Arguments
    /// * `channel_name` - Name of the channel
    ///
    /// # Returns
    /// * `Ok(Vec<(u64, u64)>)` - Byte ranges as (offset, length) tuples
    /// * `Err(MdfError)` - If channel not found or calculation fails
    pub fn get_channel_byte_ranges_by_name(&self, channel_name: &str) -> Result<Vec<(u64, u64)>> {
        let (group_index, channel_index) = self
            .find_channel_by_name_global(channel_name)
            .ok_or_else(|| {
                Error::BlockSerializationError(format!("Channel '{}' not found", channel_name))
            })?;

        self.get_channel_byte_ranges(group_index, channel_index)
    }

    /// Get channel information by name
    ///
    /// # Arguments
    /// * `channel_name` - Name of the channel
    ///
    /// # Returns
    /// * `Some((group_index, channel_index, &IndexedChannel))` - Channel info if found
    /// * `None` - If channel not found
    pub fn get_channel_info_by_name(
        &self,
        channel_name: &str,
    ) -> Option<(usize, usize, &IndexedChannel)> {
        let (group_index, channel_index) = self.find_channel_by_name_global(channel_name)?;
        let channel = self.get_channel_info(group_index, channel_index)?;
        Some((group_index, channel_index, channel))
    }
}