hpkg 0.0.8

/*
 * Copyright, 2017-2020, Alexander von Gluck IV. All rights reserved.
 * Released under the terms of the MIT license.
 *
 * vim: set noai noet ts=4 sw=4:
 *
 * Authors:
 *   Alexander von Gluck IV <kallisti5@unixzen.com>
 */

use std::fmt;
use std::path::{Path,PathBuf};
use std::fs::File;

use std::io;
use std::io::{Read,Write,Seek,SeekFrom,BufReader};
use std::error;
use std::slice;

use flate2::read::ZlibDecoder;
use zstd;

pub const MAX_TOC:				u64 = 64 * 1024 * 1024;
pub const MAX_ATTRIBUTES:		u64 = 1 * 1024 * 1024;

// HPKG attribute IDs (from Haiku's PackageAttributes.h)
const ATTR_PACKAGE_NAME: u16					= 15;
const ATTR_PACKAGE_SUMMARY: u16					= 16;
const ATTR_PACKAGE_DESCRIPTION: u16				= 17;
const ATTR_PACKAGE_VENDOR: u16					= 18;
const ATTR_PACKAGE_PACKAGER: u16				= 19;
const ATTR_PACKAGE_FLAGS: u16					= 20;
const ATTR_PACKAGE_ARCHITECTURE: u16			= 21;
const ATTR_PACKAGE_CHECKSUM: u16				= 35;
const ATTR_PACKAGE_URL: u16						= 38;
const ATTR_PACKAGE_SOURCE_URL: u16				= 39;
const ATTR_PACKAGE_INSTALL_PATH: u16			= 40;
const ATTR_PACKAGE_BASE_PACKAGE: u16			= 41;

// Attribute type constants
const HPKG_ATTR_TYPE_INT: u16		= 1;
const HPKG_ATTR_TYPE_UINT: u16		= 2;
const HPKG_ATTR_TYPE_STRING: u16	= 3;
const HPKG_ATTR_TYPE_RAW: u16		= 4;

// TOC attribute IDs (from Haiku's PackageAttributes.h)
const ATTR_DIRECTORY_ENTRY: u16		= 0;
const ATTR_FILE_TYPE: u16			= 1;
const ATTR_FILE_PERMISSIONS: u16	= 2;
const ATTR_FILE_USER: u16			= 3;
const ATTR_FILE_GROUP: u16			= 4;
const ATTR_FILE_ATIME: u16			= 5;
const ATTR_FILE_MTIME: u16			= 6;
const ATTR_FILE_CRTIME: u16			= 7;
const ATTR_DATA: u16				= 13;
const ATTR_SYMLINK_PATH: u16		= 14;

// File type constants
const HPKG_FILE_TYPE_FILE: u32		= 0;
const HPKG_FILE_TYPE_DIRECTORY: u32	= 1;
const HPKG_FILE_TYPE_SYMLINK: u32	= 2;

// Architecture enum values (from Haiku's PackageArchitecture.h)
const ARCH_ANY: u64		= 0;
const ARCH_X86: u64		= 1;
const ARCH_X86_GCC2: u64	= 2;
const ARCH_SOURCE: u64		= 3;
const ARCH_X86_64: u64		= 4;
const ARCH_PPC: u64		= 5;
const ARCH_ARM: u64		= 6;
const ARCH_M68K: u64		= 7;
const ARCH_SPARC: u64		= 8;
const ARCH_ARM64: u64		= 9;
const ARCH_RISCV64: u64		= 10;

enum BHPKGAttributeID {
	BHpkgAttributeIdDirectoryEntry,
	BHpkgAttributeIdFileType,
	BHpkgAttributeIdFilePermissions,
	BHpkgAttributeIdFileUser,
	BHpkgAttributeIdFileGroup,
	BHpkgAttributeIdFileAtime,
	BHpkgAttributeIdFileMtime,
	BHpkgAttributeIdFileCrtime,
	BHpkgAttributeIdFileAtimeNanos,
	BHpkgAttributeIdFileMtimeNanos,
	BHpkgAttributeIdFileCrtimNanos,
	BHpkgAttributeIdFileAttribute,
	BHpkgAttributeIdFileAttributeType,
	BHpkgAttributeIdData,
	BHpkgAttributeIdDataSize,
	BHpkgAttributeIdDataCompression,
	BHpkgAttributeIdDataChunkSize,
	BHpkgAttributeIdSymlinkPath,
	BHpkgAttributeIdPackageName,
	BHpkgAttributeIdPackageSummary,
	BHpkgAttributeIdPackageDescription,
	BHpkgAttributeIdPackageVendor,
	BHpkgAttributeIdPackagePackager,
	BHpkgAttributeIdPackageFlags,
	BHpkgAttributeIdPackageArchitecture,
	BHpkgAttributeIdPackageVersionMajor,
	BHpkgAttributeIdPackageVersionMinor,
	BHpkgAttributeIdPackageVersionMicro,
	BHpkgAttributeIdPackageVersionRevision,
	BHpkgAttributeIdPackageCopyright,
	BHpkgAttributeIdPackageLicense,
	BHpkgAttributeIdPackageProvides,
	BHpkgAttributeIdPackageProvidesType,
	BHpkgAttributeIdPackageRequires,
	BHpkgAttributeIdPackageSupplements,
	BHpkgAttributeIdPackageConflicts,
	BHpkgAttributeIdPackageFreshens,
	BHpkgAttributeIdPackageReplaces,
	BHpkgAttributeIdPackageResolvableOperator,
	BHpkgAttributeIdPackageChecksum,
	BHpkgAttributeIdPackageVersionPreRelease,
	BHpkgAttributeIdPackageProvidesCompatible,
	BHpkgAttributeIdPackageUrl,
	BHpkgAttributeIdPackageSourceUrl,
	BHpkgAttributeIdPackageInstallPath,
	BHpkgAttributeIdEnumCount
}

#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct PackageHeaderV2 {
	pub magic: u32,
	pub header_size: u16,
	pub version: u16,
	pub total_size: u64,
	pub minor_version: u16,

	// Heap
	pub heap_compression: u16,
	pub heap_chunk_size: u32,
	pub heap_size_compressed: u64,
	pub heap_size_uncompressed: u64,

	// package attributes section
	pub attributes_length: u32,
	pub attributes_strings_length: u32,
	pub attributes_strings_count: u32,
	pub reserved1: u32,

	// TOC section
	pub toc_length: u64,
	pub toc_strings_length: u64,
	pub toc_strings_count: u64,
}

#[derive(Debug, Clone)]
pub struct PackageFileSection {
	pub uncompressed_length: u32,
	pub data: u8,		// TODO: Data uint8*
	pub offset: u64,
	pub current_offset: u64,
	pub strings_length: u64,
	pub strings_count: u64,
	pub strings: u8,	// TODO: char**
	pub name: String,
}

/// Representation of a hpkg software archive
#[derive(Clone)]
pub struct Package {
	pub filename: Option<PathBuf>,
	pub header: Option<PackageHeaderV2>,

	pub name: Option<String>,
	pub summary: Option<String>,
	pub description: Option<String>,
	pub vendor: Option<String>,
	pub packager: Option<String>,
	pub basepackage: Option<i32>,
	pub checksum: Option<String>,
	pub installpath: Option<String>,
	pub flags: u32,
	pub architecture: Option<String>,
	pub url: Option<String>,
	pub source_url: Option<String>,

	/// Uncompressed heap data
	pub heap_data: Vec<Vec<u8>>,

	/// Files extracted from the TOC section
	pub files: Vec<FileEntry>,

	heap_chunk_offsets: Vec<u64>,
	flattened_heap: Vec<u8>,
}

/// A file or directory within an HPKG archive.
#[derive(Debug, Clone)]
pub struct FileEntry {
	pub path: String,
	pub file_type: u32,
	pub permissions: u32,
	pub user: Option<String>,
	pub group: Option<String>,
	pub modified_time: Option<u64>,
	pub symlink_path: Option<String>,
	/// Offset of file data within the flattened heap (if any)
	pub data_offset: Option<usize>,
	/// Size of file data within the flattened heap (if any)
	pub data_size: Option<usize>,
}

/// Intermediate representation of an attribute value while parsing.
#[derive(Debug)]
enum AttrValue {
	Int(i64),
	Uint(u64),
	String(String),
	Raw(Vec<u8>),
}

// ---------------------------------------------------------------------------
// LEB128 / attribute helpers
// ---------------------------------------------------------------------------

fn read_unsigned_leb128(data: &[u8], offset: &mut usize) -> Result<u64, Box<dyn error::Error>> {
	let mut result: u64 = 0;
	let mut shift = 0;
	loop {
		if *offset >= data.len() {
			return Err(From::from("Unexpected end of data while reading LEB128".to_string()));
		}
		let byte = data[*offset];
		*offset += 1;
		result |= ((byte & 0x7f) as u64) << shift;
		if byte & 0x80 == 0 {
			return Ok(result);
		}
		shift += 7;
		if shift >= 64 {
			return Err(From::from("LEB128 integer too large".to_string()));
		}
	}
}

fn decode_attribute_tag(tag: u64) -> (u16, u16, u16, bool) {
	let raw = (tag as u16).wrapping_sub(1);
	let id = raw & 0x7f;
	let type_ = (raw >> 7) & 0x7;
	let has_children = (raw >> 10) & 0x1 != 0;
	let encoding = (raw >> 11) & 0x3;
	(id, type_, encoding, has_children)
}

fn arch_to_string(value: u64) -> String {
	match value {
		ARCH_ANY => "any".to_string(),
		ARCH_X86 => "x86".to_string(),
		ARCH_X86_GCC2 => "x86_gcc2".to_string(),
		ARCH_SOURCE => "source".to_string(),
		ARCH_X86_64 => "x86_64".to_string(),
		ARCH_PPC => "ppc".to_string(),
		ARCH_ARM => "arm".to_string(),
		ARCH_M68K => "m68k".to_string(),
		ARCH_SPARC => "sparc".to_string(),
		ARCH_ARM64 => "arm64".to_string(),
		ARCH_RISCV64 => "riscv64".to_string(),
		_ => format!("arch_{}", value),
	}
}

fn read_struct<T, R: Read>(mut read: R) -> io::Result<T> {
	let num_bytes = ::std::mem::size_of::<T>();
	unsafe {
		let mut s = ::std::mem::zeroed();
		let buffer = slice::from_raw_parts_mut(&mut s as *mut T as *mut u8, num_bytes);
		match read.read_exact(buffer) {
			Ok(()) => Ok(s),
			Err(e) => {
				Err(e)
			}
		}
	}
}

impl fmt::Display for Package {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		write!(f, "package. Name {:?}, Vendor {:?}, Summary {:?}, Arch {:?}",
			self.name, self.vendor, self.summary, self.architecture)
	}
}

impl fmt::Debug for Package {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		let header = match self.header {
			Some(ref h) => h,
			None => {
				write!(f, "Haiku Package (no header loaded)")?;
				return Ok(());
			}
		};

		write!(f, "Haiku Package\n")?;

		// Internal structures
		write!(f, "Header:\n")?;
		write!(f, "        Heap chunk size: {}\n", header.heap_chunk_size)?;
		write!(f, "   Heap compressed size: {}\n", header.heap_size_compressed)?;
		write!(f, " Heap uncompressed size: {}\n", header.heap_size_uncompressed)?;

		let compression = match header.heap_compression {
			0 => "Uncompressed".to_string(),
			1 => "ZLib".to_string(),
			2 => "ZStd".to_string(),
			_ => "Unknown".to_string(),
		};
		write!(f, "       Heap compression: {}\n", compression)?;

		// External metadata from within package
		write!(f, "\nMetadata:\n")?;
		write!(f, "            name: {:?}\n", self.name)?;
		write!(f, "         summary: {:?}\n", self.summary)?;
		write!(f, "     description: {:?}\n", self.description)?;
		write!(f, "          vendor: {:?}\n", self.vendor)?;
		write!(f, "        packager: {:?}\n", self.packager)?;
		write!(f, "           flags: {}\n", self.flags)?;
		write!(f, "    architecture: {:?}\n", self.architecture)?;
		write!(f, "        checksum: {:?}\n", self.checksum)?;
		write!(f, "    install path: {:?}\n", self.installpath)?;
		write!(f, "    base package: {:?}\n", self.basepackage)?;
		write!(f, "             url: {:?}\n", self.url)?;
		write!(f, "      source url: {:?}\n", self.source_url)?;
		Ok(())
	}
}

impl Package {
	/// Create a new empty hpkg software archive representation
	pub fn new() -> Package {
		Package {
			header: None,
			filename: None,
			name: None,
			summary: None,
			description: None,
			vendor: None,
			packager: None,
			basepackage: None,
			checksum: None,
			installpath: None,
			flags: 0,
			architecture: None,
			url: None,
			source_url: None,
			heap_data: Vec::new(),
			files: Vec::new(),
			heap_chunk_offsets: Vec::new(),
			flattened_heap: Vec::new(),
		}
	}

	/// Parse the header of a hpkg and populate Package
	fn parse_header(&mut self) -> Result<(), Box<dyn error::Error>> {
		let filename = match &self.filename {
			Some(s) => s,
			None => {
				return Err(From::from(format!("Package filename missing!")));
			}
		};
		let mut f = File::open(filename)?;
		f.seek(SeekFrom::Start(0))?;
		let reader = BufReader::new(&f);

		let mut header = read_struct::<PackageHeaderV2, _>(reader)?;
		let magic_bytes = header.magic.to_ne_bytes();
		if magic_bytes != [b'h', b'p', b'k', b'g'] {
			return Err(From::from(format!("Unknown magic: {:?}", magic_bytes)));
		}

		// Endian Adjustments (are there better ways to do this?)
		header.header_size = u16::from_be(header.header_size);
		header.version = u16::from_be(header.version);
		header.total_size = u64::from_be(header.total_size);
		header.minor_version = u16::from_be(header.minor_version);
		header.heap_compression = u16::from_be(header.heap_compression);
		header.heap_chunk_size = u32::from_be(header.heap_chunk_size);
		header.heap_size_compressed = u64::from_be(header.heap_size_compressed);
		header.heap_size_uncompressed = u64::from_be(header.heap_size_uncompressed);
		header.attributes_length = u32::from_be(header.attributes_length);
		header.attributes_strings_length = u32::from_be(header.attributes_strings_length);
		header.attributes_strings_count = u32::from_be(header.attributes_strings_count);
		header.reserved1 = u32::from_be(header.reserved1);
		header.toc_length = u64::from_be(header.toc_length);
		header.toc_strings_length = u64::from_be(header.toc_strings_length);
		header.toc_strings_count = u64::from_be(header.toc_strings_count);

		// We don't really care about v1 since it saw such minor rollout
		if header.version != 2 {
			return Err(From::from(format!("Unknown hpkg version: {}", header.version)));
		}

		// If the minor version of a package/repository file is greater than the
		// current one unknown attributes are ignored without error.

		// TOC and attributes are at the end of the heap section
		if header.header_size as u64 + header.heap_size_compressed != header.total_size {
			return Err(From::from(format!("Invalid hpkg header lengths")));
		}
		self.header = Some(header);

		// Populate chunk offset table for all heap types.
		// Compressed heaps store a uint16 size table at the end of the heap;
		// uncompressed heaps have evenly-spaced chunks.
		self.heap_chunkify()?;

		Ok(())
	}

	/// Determine the offsets of each heap chunk and store them
	fn heap_chunkify(&mut self) -> Result<u64, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;

		// Extract header fields before taking any mutable borrow.
		let (heap_compression, heap_chunk_size, header_size, heap_size_compressed) = {
			let h = self.header.as_ref().unwrap();
			(h.heap_compression, h.heap_chunk_size, h.header_size, h.heap_size_compressed)
		};

		self.heap_chunk_offsets.push(0);

		if heap_compression == 0 {
			// Uncompressed heaps have no chunk size table; chunks are evenly spaced.
			for i in 1..chunks {
				self.heap_chunk_offsets
					.push(i as u64 * heap_chunk_size as u64);
			}
		} else {
			let filename = self.filename.as_ref().unwrap().clone();

			let chunk_size_table_len = (chunks - 1) * 2;
			if heap_size_compressed <= chunk_size_table_len {
				return Err(From::from(format!(
					"Compressed heap smaller than chunk size table"
				)));
			}
			let table_start =
				header_size as u64 + heap_size_compressed - chunk_size_table_len;

			let mut f = File::open(&filename)?;
			f.seek(SeekFrom::Start(table_start))?;
			let mut chunkbuffer = vec![0; chunk_size_table_len as usize];
			BufReader::new(&f).read_exact(&mut chunkbuffer)?;
			for chunk_index in 0..chunkbuffer.len() / 2 {
				let base = chunk_index * 2;
				let mut raw_cookies: u64 = ((chunkbuffer[base] as u64) << 8)
					| chunkbuffer[base + 1] as u64;
				raw_cookies += self.heap_chunk_offsets.last().unwrap() + 1;
				self.heap_chunk_offsets.push(raw_cookies as u64);
				#[cfg(test)]
				println!("{} : {}", base, raw_cookies);
			}
		}

		Ok(0)
	}

	#[cfg(test)]
	fn heap_end(&mut self) -> Result<u64, Box<dyn error::Error>> {
		let header = self.header.as_ref().unwrap();
		let end = header.header_size as u64 + header.heap_size_compressed;
		if header.heap_compression == 0 {
			return Ok(end);
		}
		// heap_size_compressed includes the chunk size table; exclude it so
		// heap_end points to the end of the actual compressed data.
		let chunks = self.heap_chunk_count()?;
		let chunk_table_len = (chunks - 1) * 2;
		Ok(end - chunk_table_len)
	}

	/// Estimate the number of heap chunks by examining the total uncompressed size
	/// vs the uncompressed heap chunk size
	fn heap_chunk_count(&mut self) -> Result<u64, Box<dyn error::Error>> {
		let header = self.header.as_ref().unwrap();
		let chunk_size = header.heap_chunk_size as u64;
		Ok((header.heap_size_uncompressed + chunk_size - 1) / chunk_size)
	}

	#[cfg(test)]
	/// Find the compressed heap chunk size via the lookup table
	fn heap_chunk_length(&mut self, index: u64) -> Result<usize, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;
		let start_offset = self.heap_chunk_offsets[index as usize] as usize;

		if index > chunks - 1 {
			return Err(From::from(format!("Index {} greater than chunk count {}!", index, chunks)));
		}

		if index < self.heap_chunk_offsets.len() as u64 - 1 {
			let next_offset = self.heap_chunk_offsets[index as usize + 1] as usize;
			return Ok(next_offset - start_offset);
		}

		return Ok(self.heap_end()? as usize - start_offset);
	}

	/// Find the offset of a heap chunk
	fn heap_chunk_offset(&mut self, index: u64) -> Result<usize, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;
		if index > chunks - 1 {
			return Err(From::from(format!("Index {} greater than chunk count {}!", index, chunks)));
		}

		let header = self.header.as_ref().unwrap();
		let start = header.header_size as usize;

		Ok(start + self.heap_chunk_offsets[index as usize] as usize)
	}

	#[cfg(test)]
	fn verify_heap_chain_sanity(&mut self) -> Result<(), Box<dyn error::Error>> {
		let heap_end = self.heap_end()?;
		let chunks = self.heap_chunk_count()? - 1;
		for index in 0..chunks {
			print!("Chunk {} of {}...", index, chunks);
			let start = self.heap_chunk_offset(index)?;
			let length = self.heap_chunk_length(index)?;
			if index < chunks {
				let next = self.heap_chunk_offset(index + 1)?;
				assert_eq!(start + length, next);
				print!("{} - {}\n", start, start + length);
			} else {
				assert_eq!(start + length, heap_end as usize);
				print!("{} - {}\n", start, heap_end);
			}
		}
		Ok(())
	}

	/// Inflate the specified heap chunk via the specified compressor
	fn inflate_heap_chunk(&mut self, index: u64) -> Result<usize, Box<dyn error::Error>> {
		let in_pos = self.heap_chunk_offset(index)?;

		// Extract header fields upfront to avoid borrow-conflicts with self mutability.
		let (heap_compression, heap_chunk_size, heap_size_compressed, heap_size_uncompressed) = {
			let h = self.header.as_ref().unwrap();
			(h.heap_compression, h.heap_chunk_size, h.heap_size_compressed,
			 h.heap_size_uncompressed)
		};
		let filename = self.filename.as_ref().unwrap().clone();
		let chunks = self.heap_chunk_count()?;
		let is_last = index == chunks - 1;

		// Determine the exact compressed size for this chunk.
		let compressed_size: usize = if heap_compression == 0 {
			0 // unused for uncompressed
		} else if !is_last {
			(self.heap_chunk_offsets[index as usize + 1]
				- self.heap_chunk_offsets[index as usize]) as usize
		} else {
			// Last chunk: compressed data ends before the chunk size table.
			let chunk_table_len = (chunks - 1) * 2;
			let total_compressed = heap_size_compressed - chunk_table_len;
			(total_compressed - self.heap_chunk_offsets[index as usize]) as usize
		};

		// Determine the uncompressed size (last chunk may be smaller).
		let uncompressed_size: usize = if !is_last {
			heap_chunk_size as usize
		} else {
			(heap_size_uncompressed - (chunks - 1) * heap_chunk_size as u64) as usize
		};

		let mut f = File::open(&filename)?;
		f.seek(SeekFrom::Start(in_pos as u64))?;

		if heap_compression == 0 {
			let mut buffer = vec![0u8; uncompressed_size];
			f.read_exact(&mut buffer)?;
			self.heap_data.push(buffer);
			Ok(uncompressed_size)
		} else {
			let mut compressed = vec![0u8; compressed_size];
			f.read_exact(&mut compressed)?;

			let mut buffer = vec![0u8; uncompressed_size];
			let mut reader: Box<dyn Read> = match heap_compression {
				1 => Box::new(ZlibDecoder::new(&compressed[..])),
				2 => Box::new(zstd::stream::read::Decoder::new(&compressed[..])?),
				_ => return Err(From::from(format!(
					"Unknown hpkg heap compression: {}", heap_compression))),
			};
			reader.read_exact(&mut buffer)?;
			self.heap_data.push(buffer);
			Ok(uncompressed_size)
		}
	}

	/// Inflate the heap section of a hpkg for later processing
	/// XXX: This will likely need reworked... just trying to figure out what's going on
	fn inflate_heap(&mut self) -> Result<usize, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;

		// Each chunk is compressed individually so each represents a separate zlib stream.
		for chunk_index in 0..chunks {
			self.inflate_heap_chunk(chunk_index)?;
		}
		Ok(0)
	}

	pub fn dump_raw_heap<P: AsRef<Path>>(&mut self, prefix: P) -> Result<usize, Box<dyn error::Error>> {
		for (index,data) in self.heap_data.iter().enumerate() {
			let mut filename = PathBuf::new();
			filename.push(prefix.as_ref());
			filename.push(format!("heap-chunk-{}.data", index));
			let mut dumpfile = File::create(filename)?;
			let mut pos = 0;

			while pos < data.len() {
				let bytes_written = dumpfile.write(&data[pos..])?;
				pos += bytes_written;
			}
		}
		Ok(0)
	}

	/// Flatten the chunked heap into a single contiguous buffer for random-access parsing.
	fn flatten_heap(&mut self) {
		let total: usize = self.heap_data.iter().map(|c| c.len()).sum();
		let mut flat = Vec::with_capacity(total);
		for chunk in &self.heap_data {
			flat.extend_from_slice(chunk);
		}
		self.flattened_heap = flat;
	}

	/// Parse the string that starts at `offset` and advance the cursor past its
	/// null terminator.  Returns the string (without the terminator).
	fn read_string_from(&self, offset: &mut usize) -> Result<&str, Box<dyn error::Error>> {
		let data = &self.flattened_heap;
		let start = *offset;
		// find null terminator
		while *offset < data.len() && data[*offset] != 0 {
			*offset += 1;
		}
		if *offset >= data.len() {
			return Err(From::from("Unexpected end of heap data in string table".to_string()));
		}
		let s = std::str::from_utf8(&data[start..*offset])?;
		*offset += 1; // skip null
		Ok(s)
	}

	/// Parse the strings subsection at the given offset, returning a vector of
	/// strings that can be referenced by index.
	fn parse_string_table(&self, offset: usize, count: u32) -> Result<Vec<String>, Box<dyn error::Error>> {
		let mut pos = offset;
		let mut table = Vec::with_capacity(count as usize);
		for _ in 0..count {
			let s = self.read_string_from(&mut pos)?.to_string();
			table.push(s);
		}
		Ok(table)
	}

	/// Read an attribute value from the flattened heap and advance `offset`.
	fn read_attr_value(&self, offset: &mut usize, type_: u16, encoding: u16,
		string_table: &[String]) -> Result<AttrValue, Box<dyn error::Error>>
	{
		match type_ {
			HPKG_ATTR_TYPE_INT | HPKG_ATTR_TYPE_UINT => {
				let v: u64 = match encoding {
					0 => {
						if *offset >= self.flattened_heap.len() {
							return Err(From::from("heap underflow reading int8".to_string()));
						}
						let b = self.flattened_heap[*offset];
						*offset += 1;
						b as u64
					}
					1 => {
						if *offset + 2 > self.flattened_heap.len() {
							return Err(From::from("heap underflow reading int16".to_string()));
						}
						let v = u16::from_be_bytes(
							self.flattened_heap[*offset..*offset + 2].try_into().unwrap());
						*offset += 2;
						v as u64
					}
					2 => {
						if *offset + 4 > self.flattened_heap.len() {
							return Err(From::from("heap underflow reading int32".to_string()));
						}
						let v = u32::from_be_bytes(
							self.flattened_heap[*offset..*offset + 4].try_into().unwrap());
						*offset += 4;
						v as u64
					}
					3 => {
						if *offset + 8 > self.flattened_heap.len() {
							return Err(From::from("heap underflow reading int64".to_string()));
						}
						let v = u64::from_be_bytes(
							self.flattened_heap[*offset..*offset + 8].try_into().unwrap());
						*offset += 8;
						v
					}
					_ => return Err(From::from(format!("Unknown int encoding {}", encoding))),
				};
				if type_ == HPKG_ATTR_TYPE_INT {
					Ok(AttrValue::Int(v as i64))
				} else {
					Ok(AttrValue::Uint(v))
				}
			}

			HPKG_ATTR_TYPE_STRING => {
				let s = if encoding == 0 {
					self.read_string_from(offset)?.to_string()
				} else {
					let idx = read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
					if idx >= string_table.len() {
						return Err(From::from(format!("String table index {} out of bounds", idx)));
					}
					string_table[idx].clone()
				};
				Ok(AttrValue::String(s))
			}

			HPKG_ATTR_TYPE_RAW => {
				let size = read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
				if encoding == 0 {
					if *offset + size > self.flattened_heap.len() {
						return Err(From::from("heap underflow reading raw inline data".to_string()));
					}
					let data = self.flattened_heap[*offset..*offset + size].to_vec();
					*offset += size;
					Ok(AttrValue::Raw(data))
				} else {
					let heap_offset = read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
					if *offset > self.flattened_heap.len() || heap_offset + size > self.flattened_heap.len() {
						return Err(From::from("Invalid raw data reference into heap".to_string()));
					}
					let data = self.flattened_heap[heap_offset..heap_offset + size].to_vec();
					Ok(AttrValue::Raw(data))
				}
			}

			_ => Err(From::from(format!("Unknown attribute type {}", type_))),
		}
	}

	/// Recursively walk attribute trees in a section of the flattened heap and
	/// populate metadata fields on this Package.
	fn parse_attributes_inner(&mut self, offset: &mut usize,
		string_table: &[String], depth: usize) -> Result<(), Box<dyn error::Error>>
	{
		loop {
			if *offset >= self.flattened_heap.len() {
				return Ok(());
			}

			let tag_raw = read_unsigned_leb128(&self.flattened_heap, offset)?;
			if tag_raw == 0 {
				// End-of-children marker
				return Ok(());
			}

			let (id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);
			let value = self.read_attr_value(offset, type_, encoding, string_table)?;

			if depth == 0 {
				// Top-level attributes only – this is the package attributes
				// section, not the TOC.
				match id {
					ATTR_PACKAGE_NAME => {
						if let AttrValue::String(s) = &value {
							self.name = Some(s.clone());
						}
					}
					ATTR_PACKAGE_SUMMARY => {
						if let AttrValue::String(s) = &value {
							self.summary = Some(s.clone());
						}
					}
					ATTR_PACKAGE_DESCRIPTION => {
						if let AttrValue::String(s) = &value {
							self.description = Some(s.clone());
						}
					}
					ATTR_PACKAGE_VENDOR => {
						if let AttrValue::String(s) = &value {
							self.vendor = Some(s.clone());
						}
					}
					ATTR_PACKAGE_PACKAGER => {
						if let AttrValue::String(s) = &value {
							self.packager = Some(s.clone());
						}
					}
					ATTR_PACKAGE_FLAGS => {
						if let AttrValue::Uint(v) = &value {
							self.flags = *v as u32;
						} else if let AttrValue::Int(v) = &value {
							self.flags = *v as u32;
						}
					}
					ATTR_PACKAGE_ARCHITECTURE => {
						if let AttrValue::Uint(v) = &value {
							self.architecture = Some(arch_to_string(*v));
						} else if let AttrValue::Int(v) = &value {
							self.architecture = Some(arch_to_string(*v as u64));
						}
					}
					ATTR_PACKAGE_CHECKSUM => {
						if let AttrValue::String(s) = &value {
							self.checksum = Some(s.clone());
						}
					}
					ATTR_PACKAGE_INSTALL_PATH => {
						if let AttrValue::String(s) = &value {
							self.installpath = Some(s.clone());
						}
					}
					ATTR_PACKAGE_URL => {
						if let AttrValue::String(s) = &value {
							self.url = Some(s.clone());
						}
					}
					ATTR_PACKAGE_SOURCE_URL => {
						if let AttrValue::String(s) = &value {
							self.source_url = Some(s.clone());
						}
					}
					ATTR_PACKAGE_BASE_PACKAGE => {
						if let AttrValue::String(s) = &value {
							self.basepackage = Some(s.parse().unwrap_or(0));
						}
					}
					_ => {}
				}
			}

			if has_children {
				self.parse_attributes_inner(offset, string_table, depth + 1)?;
			}
		}
	}

	/// Parse the package attributes section (at the end of the uncompressed
	/// heap) and populate metadata fields.
	fn parse_attributes(&mut self) -> Result<(), Box<dyn error::Error>> {
		let header = self.header.as_ref().ok_or("No header loaded")?;
		if header.attributes_length == 0 {
			return Ok(());
		}
		let heap_size = header.heap_size_uncompressed as usize;
		if heap_size != self.flattened_heap.len() {
			return Err(From::from(format!(
				"Heap size mismatch: header says {} but flattened heap is {}",
				heap_size, self.flattened_heap.len())));
		}

		let attr_offset = heap_size - header.attributes_length as usize;
		let strings_len = header.attributes_strings_length as usize;

		// Strings subsection
		let strings_offset = attr_offset;
		let main_offset = attr_offset + strings_len;

		let strings = self.parse_string_table(strings_offset, header.attributes_strings_count)?;

		// Attribute tree
		let mut pos = main_offset;
		self.parse_attributes_inner(&mut pos, &strings, 0)?;

		Ok(())
	}

	/// Parse the TOC section (which sits between the heap data and the
	/// attributes section) and populate self.files.
	fn parse_toc(&mut self) -> Result<(), Box<dyn error::Error>> {
		let header = self.header.as_ref().ok_or("No header loaded")?;
		if header.toc_length == 0 {
			return Ok(());
		}
		let heap_size = header.heap_size_uncompressed as usize;
		let attr_len = header.attributes_length as usize;
		let toc_len = header.toc_length as usize;

		if toc_len + attr_len > heap_size {
			return Err(From::from("TOC + attributes overflow heap size".to_string()));
		}
		let toc_offset = heap_size - attr_len - toc_len;
		let main_offset = toc_offset + header.toc_strings_length as usize;

		let strings = self.parse_string_table(
			toc_offset, header.toc_strings_count as u32)?;

		let mut pos = main_offset;
		let mut root_path = String::new();
		self.parse_toc_entries(&mut pos, &strings, &mut root_path)?;

		Ok(())
	}

	/// Recursively walk DIRECTORY_ENTRY attributes building full paths and
	/// populating self.files.
	fn parse_toc_entries(&mut self, offset: &mut usize,
		strings: &[String], parent_path: &str) -> Result<(), Box<dyn error::Error>>
	{
		loop {
			if *offset >= self.flattened_heap.len() {
				return Ok(());
			}
			let tag_raw = read_unsigned_leb128(&self.flattened_heap, offset)?;
			if tag_raw == 0 {
				return Ok(());
			}
			let (id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);
			if id != ATTR_DIRECTORY_ENTRY {
				// Skip non-directory-entry attributes at this level
				self.skip_attribute_value(offset, type_, encoding, strings)?;
				if has_children {
					self.skip_attribute_tree(offset)?;
				}
				continue;
			}
			// Read the entry name
			let name = if let AttrValue::String(s) =
				self.read_attr_value(offset, type_, encoding, strings)?
			{
				s
			} else {
				continue;
			};
			let path = if parent_path.is_empty() {
				name
			} else {
				format!("{}/{}", parent_path, name)
			};

			// Collect children's metadata
			let mut file_type = HPKG_FILE_TYPE_DIRECTORY;
			let mut permissions: u32 = 0o644;
			let mut user: Option<String> = None;
			let mut group: Option<String> = None;
			let mut modified_time: Option<u64> = None;
			let mut symlink_path: Option<String> = None;
			let mut data_offset: Option<usize> = None;
			let mut data_size: Option<usize> = None;
			let mut sub_entries: Vec<(usize, Vec<String>)> = Vec::new();

			if has_children {
				self.parse_toc_entry_children(offset, strings, &path,
					&mut file_type, &mut permissions,
					&mut user, &mut group,
					&mut modified_time, &mut symlink_path,
					&mut data_offset, &mut data_size,
					&mut sub_entries)?;
			}

			// Determine file_type from children presence if not set explicitly
			if file_type == HPKG_FILE_TYPE_DIRECTORY && symlink_path.is_some() {
				file_type = HPKG_FILE_TYPE_SYMLINK;
			} else if file_type == HPKG_FILE_TYPE_DIRECTORY && data_offset.is_some() {
				file_type = HPKG_FILE_TYPE_FILE;
			}

			self.files.push(FileEntry {
				path: path.clone(),
				file_type,
				permissions,
				user,
				group,
				modified_time,
				symlink_path,
				data_offset,
				data_size,
			});

			// Recursively process subdirectory entries
			for (mut child_pos, _) in sub_entries {
				self.parse_toc_entries(&mut (child_pos), strings, &path)?;
			}
		}
	}

	/// Process the children of a DIRECTORY_ENTRY, extracting metadata and
	/// recording the positions of sub-directory entries for later recursion.
	#[allow(clippy::too_many_arguments)]
	fn parse_toc_entry_children(&mut self, offset: &mut usize,
		strings: &[String], _current_path: &str,
		file_type: &mut u32, permissions: &mut u32,
		user: &mut Option<String>, group: &mut Option<String>,
		modified_time: &mut Option<u64>,
		symlink_path: &mut Option<String>,
		data_offset: &mut Option<usize>, data_size: &mut Option<usize>,
		sub_entries: &mut Vec<(usize, Vec<String>)>)
		-> Result<(), Box<dyn error::Error>>
	{
		loop {
			if *offset >= self.flattened_heap.len() {
				return Ok(());
			}
			let tag_raw = read_unsigned_leb128(&self.flattened_heap, offset)?;
			if tag_raw == 0 {
				return Ok(());
			}
			let (id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);

			match id {
				ATTR_DIRECTORY_ENTRY => {
					// Sub-directory entry — record position for later recursion
					let save_pos = *offset;
					// Read the name (to advance past it) then skip if leaf
					let name = if let Ok(AttrValue::String(s)) =
						self.read_attr_value(offset, type_, encoding, strings)
					{
						s
					} else {
						continue;
					};
					if has_children {
						let mut child_names = Vec::new();
						child_names.push(name);
						sub_entries.push((save_pos, child_names));
						// Skip the children for now (we'll recurse later from
						// the saved position)
						self.skip_attribute_tree(offset)?;
					}
				}
				ATTR_FILE_TYPE => {
					if let Ok(v) = self.read_attr_value(offset, type_, encoding, strings) {
						match v {
							AttrValue::Uint(v) => *file_type = v as u32,
							AttrValue::Int(v) => *file_type = v as u32,
							_ => {}
						}
					}
				}
				ATTR_FILE_PERMISSIONS => {
					if let Ok(v) = self.read_attr_value(offset, type_, encoding, strings) {
						match v {
							AttrValue::Uint(v) => *permissions = v as u32,
							AttrValue::Int(v) => *permissions = v as u32,
							_ => {}
						}
					}
				}
				ATTR_FILE_USER => {
					if let Ok(AttrValue::String(s)) =
						self.read_attr_value(offset, type_, encoding, strings)
					{
						*user = Some(s);
					}
				}
				ATTR_FILE_GROUP => {
					if let Ok(AttrValue::String(s)) =
						self.read_attr_value(offset, type_, encoding, strings)
					{
						*group = Some(s);
					}
				}
				ATTR_FILE_MTIME => {
					if let Ok(v) = self.read_attr_value(offset, type_, encoding, strings) {
						match v {
							AttrValue::Uint(v) => *modified_time = Some(v),
							AttrValue::Int(v) => *modified_time = Some(v as u64),
							_ => {}
						}
					}
				}
				ATTR_DATA => {
					if let Ok(AttrValue::Raw(data)) =
						self.read_attr_value(offset, type_, encoding, strings)
					{
						// Store the data as-is from the raw attribute value.
						// For inline data, it's the bytes directly.
						// For heap-referenced data, it was already copied.
						let start = self.flattened_heap.len();
						self.flattened_heap.extend_from_slice(&data);
						*data_offset = Some(start);
						*data_size = Some(data.len());
					}
				}
				ATTR_SYMLINK_PATH => {
					if let Ok(AttrValue::String(s)) =
						self.read_attr_value(offset, type_, encoding, strings)
					{
						*symlink_path = Some(s);
						*file_type = HPKG_FILE_TYPE_SYMLINK;
					}
				}
				_ => {
					self.skip_attribute_value(offset, type_, encoding, strings)?;
					if has_children {
						self.skip_attribute_tree(offset)?;
					}
				}
			}
		}
	}

	/// Skip the value of an attribute without storing it.
	fn skip_attribute_value(&self, offset: &mut usize, type_: u16,
		encoding: u16, _strings: &[String]) -> Result<(), Box<dyn error::Error>>
	{
		match type_ {
			HPKG_ATTR_TYPE_INT | HPKG_ATTR_TYPE_UINT => {
				match encoding {
					0 => *offset += 1,
					1 => *offset += 2,
					2 => *offset += 4,
					3 => *offset += 8,
					_ => return Err(From::from("Unknown int encoding")),
				}
			}
			HPKG_ATTR_TYPE_STRING => {
				if encoding == 0 {
					// Inline null-terminated string
					while *offset < self.flattened_heap.len()
						&& self.flattened_heap[*offset] != 0
					{
						*offset += 1;
					}
					if *offset < self.flattened_heap.len() {
						*offset += 1; // skip null
					}
				} else {
					// String table index
					read_unsigned_leb128(&self.flattened_heap, offset)?;
				}
			}
			HPKG_ATTR_TYPE_RAW => {
				let size = read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
				if encoding == 0 {
					*offset += size;
				} else {
					// Heap reference: read LEB128 offset but skip the data
					read_unsigned_leb128(&self.flattened_heap, offset)?;
				}
			}
			_ => {}
		}
		Ok(())
	}

	/// Skip an entire subtree of attributes (until the end-of-children marker).
	fn skip_attribute_tree(&self, offset: &mut usize) -> Result<(), Box<dyn error::Error>> {
		let mut depth = 0usize;
		loop {
			if *offset >= self.flattened_heap.len() {
				return Ok(());
			}
			let tag_raw = read_unsigned_leb128(&self.flattened_heap, offset)?;
			if tag_raw == 0 {
				if depth == 0 {
					return Ok(());
				}
				depth -= 1;
				continue;
			}
			let (_id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);
			self.skip_attribute_value(offset, type_, encoding, &[])?;
			if has_children {
				depth += 1;
			}
		}
	}

	/// Return the contents of the file at the given path, or an error if not
	/// found.
	pub fn read_file(&self, path: &str) -> Result<&[u8], Box<dyn error::Error>> {
		let norm = path.trim_start_matches('/');
		for entry in &self.files {
			if entry.path == norm && entry.file_type == HPKG_FILE_TYPE_FILE {
				if let (Some(offset), Some(size)) = (entry.data_offset, entry.data_size) {
					if offset + size > self.flattened_heap.len() {
						return Err(From::from("File data out of bounds".to_string()));
					}
					return Ok(&self.flattened_heap[offset..offset + size]);
				}
				return Err(From::from("File has no data".to_string()));
			}
		}
		Err(From::from(format!("File not found: {}", path)))
	}

	/// Return a list of all files and directories in the package.
	pub fn list_files(&self) -> &[FileEntry] {
		&self.files
	}

	/// Section start calculated as endOffset - section length
	///   Attributes Section = uncompressed heap size - attributes section length
	///   TOC Section = Attributes Section offset - toc section length

	/// Open an hpkg file produce a populated Package representation
	pub fn load<P: AsRef<Path>>(hpkg_file: P)
		-> Result<Package, Box<dyn error::Error>> {

		let mut f = File::open(hpkg_file.as_ref())?;
		f.seek(SeekFrom::Start(0))?;

		let mut hpkg = Package::new();
		hpkg.filename = Some(hpkg_file.as_ref().to_path_buf());
		hpkg.parse_header()?;
		hpkg.inflate_heap()?;
		hpkg.flatten_heap();
		hpkg.parse_attributes()?;
		hpkg.parse_toc()?;

		return Ok(hpkg);
	}
}

#[cfg(test)]
mod tests {
	use super::*;
	//use std::str::FromStr;

	#[test]
	/// Test creating a new empty package definition
	fn test_package_new() {
		let _package = Package::new();
	}

	#[test]
	/// Test loading a valid package from disk
	fn test_package_load_valid() {
		let hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		assert!(hpkg.header.is_some());
	}

	#[test]
	/// Test loading an invalid package from disk
	fn test_package_load_invalid() {
		assert!(Package::load("sample/source-5.8-5-source.hpkg").is_err());
	}

	#[test]
	/// Test total size compared to header
	fn test_package_total_size() {
		let metadata = match std::fs::metadata("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		let hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		let header = match hpkg.header {
			Some(o) => o,
			None => {
				println!("ERROR: Invalid Header!");
				assert!(false);
				return;
			},
		};
		assert_eq!(metadata.len(), header.total_size);
	}

	#[test]
	/// Test displaying package information
	fn test_package_dump_info() {
		let hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		println!("{}", hpkg);
		println!("{:?}", hpkg);
		assert_eq!(hpkg.name.as_deref(), Some("ctags_source"));
		assert_eq!(hpkg.vendor.as_deref(), Some("Haiku Project"));
		assert_eq!(hpkg.summary.as_deref(), Some("A tool that creates tags files for code browsing in editors (source files)"));
		assert_eq!(hpkg.architecture.as_deref(), Some("source"));
		assert_eq!(hpkg.url.as_deref(), Some("http://ctags.sourceforge.net/"));
		assert_eq!(hpkg.source_url.as_deref(), Some("https://ports-mirror.haiku-os.org/ctags/ctags-5.8.tar.gz"));
	}

	#[test]
	/// Test listing and reading files from the package
	fn test_package_read_files() {
		let hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		let files = hpkg.list_files();
		assert!(files.len() > 0, "Package should contain files");

		// Log the first 10 files for debugging
		for entry in files.iter().take(10) {
			println!("  {} (type={}, size={:?})",
				entry.path, entry.file_type, entry.data_size);
		}

		// Every file with data should be readable
		for entry in files.iter().filter(|f| f.data_size.is_some()) {
			let contents = hpkg.read_file(&entry.path);
			assert!(contents.is_ok(),
				"Should read file '{}': {:?}", entry.path, contents);
			if let Ok(data) = contents {
				assert_eq!(data.len(), entry.data_size.unwrap(),
					"File '{}' size mismatch", entry.path);
			}
		}
	}
}