hpkg 0.0.3

/*
 * Copyright, 2017-2020, Alexander von Gluck IV. All rights reserved.
 * Released under the terms of the MIT license.
 *
 * vim: set noai noet ts=4 sw=4:
 *
 * Authors:
 *   Alexander von Gluck IV <kallisti5@unixzen.com>
 */

use std::fmt;
use std::path::{Path,PathBuf};
use std::fs::File;

use std::io;
use std::io::{Read,Seek,SeekFrom,BufReader};
use std::error;
use std::slice;

use flate2::read::ZlibDecoder;
use zstd;

pub const MAX_TOC:			u64 = 64 * 1024 * 1024;
pub const MAX_ATTRIBUTES:	u64 = 1 * 1204 * 1024;

enum BHPKGAttributeID {
	BHpkgAttributeIdDirectoryEntry,
	BHpkgAttributeIdFileType,
	BHpkgAttributeIdFilePermissions,
	BHpkgAttributeIdFileUser,
	BHpkgAttributeIdFileGroup,
	BHpkgAttributeIdFileAtime,
	BHpkgAttributeIdFileMtime,
	BHpkgAttributeIdFileCrtime,
	BHpkgAttributeIdFileAtimeNanos,
	BHpkgAttributeIdFileMtimeNanos,
	BHpkgAttributeIdFileCrtimNanos,
	BHpkgAttributeIdFileAttribute,
	BHpkgAttributeIdFileAttributeType,
	BHpkgAttributeIdData,
	BHpkgAttributeIdDataSize,
	BHpkgAttributeIdDataCompression,
	BHpkgAttributeIdDataChunkSize,
	BHpkgAttributeIdSymlinkPath,
	BHpkgAttributeIdPackageName,
	BHpkgAttributeIdPackageSummary,
	BHpkgAttributeIdPackageDescription,
	BHpkgAttributeIdPackageVendor,
	BHpkgAttributeIdPackagePackager,
	BHpkgAttributeIdPackageFlags,
	BHpkgAttributeIdPackageArchitecture,
	BHpkgAttributeIdPackageVersionMajor,
	BHpkgAttributeIdPackageVersionMinor,
	BHpkgAttributeIdPackageVersionMicro,
	BHpkgAttributeIdPackageVersionRevision,
	BHpkgAttributeIdPackageCopyright,
	BHpkgAttributeIdPackageLicense,
	BHpkgAttributeIdPackageProvides,
	BHpkgAttributeIdPackageProvidesType,
	BHpkgAttributeIdPackageRequires,
	BHpkgAttributeIdPackageSupplements,
	BHpkgAttributeIdPackageConflicts,
	BHpkgAttributeIdPackageFreshens,
	BHpkgAttributeIdPackageReplaces,
	BHpkgAttributeIdPackageResolvableOperator,
	BHpkgAttributeIdPackageChecksum,
	BHpkgAttributeIdPackageVersionPreRelease,
	BHpkgAttributeIdPackageProvidesCompatible,
	BHpkgAttributeIdPackageUrl,
	BHpkgAttributeIdPackageSourceUrl,
	BHpkgAttributeIdPackageInstallPath,
	BHpkgAttributeIdEnumCount
}

#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct PackageHeaderV2 {
	pub magic: u32,
	pub header_size: u16,
	pub version: u16,
	pub total_size: u64,
	pub minor_version: u16,

	// Heap
	pub heap_compression: u16,
	pub heap_chunk_size: u32,
	pub heap_size_compressed: u64,
	pub heap_size_uncompressed: u64,

	// package attributes section
	pub attributes_length: u32,
	pub attributes_strings_length: u32,
	pub attributes_strings_count: u32,
	pub reserved1: u32,

	// TOC section
	pub toc_length: u64,
	pub toc_strings_length: u64,
	pub toc_strings_count: u64,
}

#[derive(Debug, Clone)]
pub struct PackageFileSection {
	pub uncompressed_length: u32,
	pub data: u8,		// TODO: Data uint8*
	pub offset: u64,
	pub current_offset: u64,
	pub strings_length: u64,
	pub strings_count: u64,
	pub strings: u8,	// TODO: char**
	pub name: String,
}

/// Representation of a hpkg software archive
#[derive(Clone)]
pub struct Package {
	pub filename: Option<PathBuf>,
	pub header: Option<PackageHeaderV2>,

	pub name: Option<String>,
	pub summary: Option<String>,
	pub description: Option<String>,
	pub vendor: Option<String>,
	pub packager: Option<String>,
	pub basepackage: Option<i32>,
	pub checksum: Option<String>,
	pub installpath: Option<String>,
	pub flags: u32,
	pub architecture: Option<String>,

	/// Uncompressed heap data
	heap_data: Vec<Vec<u8>>,

	heap_chunk_offsets: Vec<u64>,
}

fn read_struct<T, R: Read>(mut read: R) -> io::Result<T> {
	let num_bytes = ::std::mem::size_of::<T>();
	unsafe {
		let mut s = ::std::mem::zeroed();
		let buffer = slice::from_raw_parts_mut(&mut s as *mut T as *mut u8, num_bytes);
		match read.read_exact(buffer) {
			Ok(()) => Ok(s),
			Err(e) => {
				Err(e)
			}
		}
	}
}

impl fmt::Display for Package {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		write!(f, "package. Name {:?}, Vendor {:?}, Summary {:?}, Arch {:?}",
			self.name, self.vendor, self.summary, self.architecture)
	}
}

impl fmt::Debug for Package {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
		let header = self.header.unwrap();
		//let chunks = self.heap_chunk_count();

		write!(f, "Haiku Package\n")?;

		// Internal structures
		write!(f, "Header:\n")?;
		write!(f, "        Heap chunk size: {}\n", header.heap_chunk_size)?;
		//write!(f, "       Heap chunk count: {:?}\n", chunks)?;
		write!(f, "   Heap compressed size: {}\n", header.heap_size_compressed)?;
		write!(f, " Heap uncompressed size: {}\n", header.heap_size_uncompressed)?;

		let compression = match header.heap_compression {
			0 => "Uncompressed".to_string(),
			1 => "ZLib".to_string(),
			2 => "ZStd".to_string(),
			_ => "Unknown".to_string(),
		};
		write!(f, "       Heap compression: {}\n", compression)?;

		// External metadata from within package
		write!(f, "\nMetadata:\n")?;
		write!(f, "        name: {:?}\n", self.name)?;
		write!(f, "      vendor: {:?}\n", self.vendor)?;
		write!(f, "     summary: {:?}\n", self.vendor)?;
		write!(f, "architecture: {:?}\n", self.architecture)?;
		Ok(())
	}
}

impl Package {
	/// Create a new empty hpkg software archive representation
	pub fn new() -> Package {
		Package {
			header: None,
			filename: None,
			name: None,
			summary: None,
			description: None,
			vendor: None,
			packager: None,
			basepackage: None,
			checksum: None,
			installpath: None,
			flags: 0,
			architecture: None,
			heap_data: Vec::new(),
			heap_chunk_offsets: Vec::new(),
		}
	}

	/// Parse the header of a hpkg and populate Package
	fn parse_header(&mut self) -> Result<(), Box<dyn error::Error>> {
		let filename = match &self.filename {
			Some(s) => s,
			None => {
				return Err(From::from(format!("Package filename missing!")));
			}
		};
		let mut f = File::open(filename)?;
		f.seek(SeekFrom::Start(0))?;
		let reader = BufReader::new(&f);

		let mut header = read_struct::<PackageHeaderV2, _>(reader)?;
		let magic_bytes = header.magic.to_ne_bytes();
		if magic_bytes != [b'h', b'p', b'k', b'g'] {
			return Err(From::from(format!("Unknown magic: {:?}", magic_bytes)));
		}

		// Endian Adjustments (are there better ways to do this?)
		header.header_size = u16::from_be(header.header_size);
		header.version = u16::from_be(header.version);
		header.total_size = u64::from_be(header.total_size);
		header.minor_version = u16::from_be(header.minor_version);
		header.heap_compression = u16::from_be(header.heap_compression);
		header.heap_chunk_size = u32::from_be(header.heap_chunk_size);
		header.heap_size_compressed = u64::from_be(header.heap_size_compressed);
		header.heap_size_uncompressed = u64::from_be(header.heap_size_uncompressed);
		header.attributes_length = u32::from_be(header.attributes_length);
		header.attributes_strings_length = u32::from_be(header.attributes_strings_length);
		header.attributes_strings_count = u32::from_be(header.attributes_strings_count);
		header.reserved1 = u32::from_be(header.reserved1);
		header.toc_length = u64::from_be(header.toc_length);
		header.toc_strings_length = u64::from_be(header.toc_strings_length);
		header.toc_strings_count = u64::from_be(header.toc_strings_count);

		// We don't really care about v1 since it saw such minor rollout
		if header.version != 2 {
			return Err(From::from(format!("Unknown hpkg version: {}", header.version)));
		}

		// If the minor version of a package/repository file is greater than the
		// current one unknown attributes are ignored without error.

		// TOC and attributes are at the end of the heap section
		if header.header_size as u64 + header.heap_size_compressed != header.total_size {
			return Err(From::from(format!("Invalid hpkg header lengths")));
		}
		// Give it away
		self.header = Some(header);
		// Take it back lol
		let header = self.header.as_ref().unwrap();

		// If compression was used, grab the heap size table at the end of the heap.
		if header.heap_compression != 0 {
			self.heap_chunkify()?;
			#[cfg(test)]
			self.verify_heap_chain_sanity()?;
		}

		Ok(())
	}

	/// Determine the offsets of each heap chunk and store them
	fn heap_chunkify(&mut self) -> Result<u64, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;
		let header = self.header.as_ref().unwrap();
		let filename = self.filename.as_ref().unwrap();

		let chunk_size_table_len = (chunks - 1) * 2;
		if header.heap_size_compressed <= chunk_size_table_len {
			return Err(From::from(format!("Compressed heaps smaller than chunk size table")));
		}
		let mut table_start = header.header_size as u64 + header.heap_size_compressed;
		table_start -= chunk_size_table_len;

		// First offset always zero
		self.heap_chunk_offsets.push(0);

		let mut f = File::open(filename)?;
		f.seek(SeekFrom::Start(table_start))?;
		let mut chunkbuffer = vec![0; chunk_size_table_len as usize];
		BufReader::new(&f).read(&mut chunkbuffer)?;
		for chunk_index in 0..chunkbuffer.len() / 2 {
			let base = chunk_index * 2;
			let mut raw_cookies: u64 = ((chunkbuffer[base] as u64) << 8) | chunkbuffer[base + 1] as u64;
			raw_cookies += self.heap_chunk_offsets.last().unwrap() + 1;
			self.heap_chunk_offsets.push(raw_cookies as u64);
			#[cfg(test)]
			println!("{} : {}", base, raw_cookies);
		}
		Ok(0)
	}

	#[cfg(test)]
	fn heap_end(&mut self) -> Result<u64, Box<dyn error::Error>> {
		let header = self.header.as_ref().unwrap();
		Ok(header.header_size as u64 + header.heap_size_compressed)
	}

	/// Estimate the number of heap chunks by examining the total uncompressed size
	/// vs the uncompressed heap chunk size
	fn heap_chunk_count(&mut self) -> Result<u64, Box<dyn error::Error>> {
		let header = self.header.as_ref().unwrap();
		let chunk_size = header.heap_chunk_size as u64;
		Ok((header.heap_size_uncompressed + chunk_size - 1) / chunk_size)
	}

	#[cfg(test)]
	/// Find the compressed heap chunk size via the lookup table
	fn heap_chunk_length(&mut self, index: u64) -> Result<usize, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;
		let start_offset = self.heap_chunk_offsets[index as usize] as usize;

		if index > chunks - 1 {
			return Err(From::from(format!("Index {} greater than chunk count {}!", index, chunks)));
		}

		if index < self.heap_chunk_offsets.len() as u64 - 1 {
			let next_offset = self.heap_chunk_offsets[index as usize + 1] as usize;
			return Ok(next_offset - start_offset);
		}

		return Ok(self.heap_end()? as usize - start_offset);
	}

	/// Find the offset of a heap chunk
	fn heap_chunk_offset(&mut self, index: u64) -> Result<usize, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()?;
		if index > chunks - 1 {
			return Err(From::from(format!("Index {} greater than chunk count {}!", index, chunks)));
		}

		let header = self.header.as_ref().unwrap();
		let start = header.header_size as usize;

		Ok(start + self.heap_chunk_offsets[index as usize] as usize)
	}

	#[cfg(test)]
	fn verify_heap_chain_sanity(&mut self) -> Result<(), Box<dyn error::Error>> {
		let heap_end = self.heap_end()?;
		let chunks = self.heap_chunk_count()? - 1;
		for index in 0..chunks {
			print!("Chunk {} of {}...", index, chunks);
			let start = self.heap_chunk_offset(index)?;
			let length = self.heap_chunk_length(index)?;
			if index < chunks {
				let next = self.heap_chunk_offset(index + 1)?;
				assert_eq!(start + length, next);
				print!("{} - {}\n", start, start + length);
			} else {
				assert_eq!(start + length, heap_end as usize);
				print!("{} - {}\n", start, heap_end);
			}
		}
		Ok(())
	}

	/// Inflate the specified heap chunk via the specified compressor
	fn inflate_heap_chunk(&mut self, index: u64) -> Result<usize, Box<dyn error::Error>> {
		let in_pos = self.heap_chunk_offset(index)?;
		//let in_len = self.heap_chunk_length(index)?;
		let header = self.header.as_ref().unwrap();
		let filename = self.filename.as_ref().unwrap();

		//println!("Heap chunk {}; {} - {} inflate to {}", index, in_pos, in_pos + in_len,
		//	header.heap_chunk_size);

		let mut f = File::open(filename)?;
		f.seek(SeekFrom::Start(in_pos as u64))?;

		let mut buffer = vec![0; header.heap_chunk_size as usize];
		let mut reader: Box<dyn Read> = match header.heap_compression {
			0 => Box::new(&f),
			1 => Box::new(ZlibDecoder::new(&f)),
			2 => Box::new(zstd::stream::read::Decoder::new(&f)?),
			_ => return Err(From::from(format!("Unknown hpkg heap compression: {}", header.heap_compression)))
		};

		reader.read(&mut buffer)?;
		self.heap_data.push(buffer);
		// TODO: We might want to read the actual size of this expanded chunk :D
		Ok(header.heap_chunk_size as usize)
	}

	/// Inflate the heap section of a hpkg for later processing
	/// XXX: This will likely need reworked... just trying to figure out what's going on
	fn inflate_heap(&mut self) -> Result<usize, Box<dyn error::Error>> {
		let chunks = self.heap_chunk_count()? - 1;

		// Each chunk is compressed individually so each represents a separate zlib stream.
		for chunk_index in 0..chunks {
			self.inflate_heap_chunk(chunk_index)?;
		}

		#[cfg(feature = "heap_dump")]
		self.dump_heap();

		Ok(0)
	}

	#[cfg(feature = "heap_dump")]
	fn dump_heap(&mut self) -> Result<usize, Box<dyn error::Error>> {
		println!("Dumping uncompressed heap chunks to disk...");
		for (index,data) in self.heap_data.iter().enumerate() {
			let mut dumpfile = File::create(format!("heap-chunk-{}.data", index))?;
			let mut pos = 0;

			while pos < data.len() {
				let bytes_written = dumpfile.write(&data[pos..])?;
				pos += bytes_written;
			}
		}
		Ok(0)
	}

	/// Section start calculated as endOffset - section length
	///   Attributes Section = uncompressed heap size - attributes section length
	///   TOC Section = Attributes Section offset - toc section length

	/// Open an hpkg file produce a populated Package representation
	pub fn load<P: AsRef<Path>>(hpkg_file: P)
		-> Result<Package, Box<dyn error::Error>> {

		let mut f = File::open(hpkg_file.as_ref())?;
		f.seek(SeekFrom::Start(0))?;

		let mut hpkg = Package::new();
		hpkg.filename = Some(hpkg_file.as_ref().to_path_buf());
		hpkg.parse_header()?;
		hpkg.inflate_heap()?;

		return Ok(hpkg);
	}
}

#[cfg(test)]
mod tests {
	use super::*;
	//use std::str::FromStr;

	#[test]
	/// Test creating a new empty package definition
	fn test_package_new() {
		let _package = Package::new();
	}

	#[test]
	/// Test loading a valid package from disk
	fn test_package_load_valid() {
		let hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		assert!(hpkg.header.is_some());
	}

	#[test]
	/// Test loading an invalid package from disk
	fn test_package_load_invalid() {
		assert!(Package::load("sample/source-5.8-5-source.hpkg").is_err());
	}

	#[test]
	/// Test total size compared to header
	fn test_package_total_size() {
		let metadata = match std::fs::metadata("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		let hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		let header = match hpkg.header {
			Some(o) => o,
			None => {
				println!("ERROR: Invalid Header!");
				assert!(false);
				return;
			},
		};
		assert_eq!(metadata.len(), header.total_size);
	}

	#[test]
	/// Test displaying package information
	fn test_package_dump_info() {
		let _hpkg = match Package::load("sample/ctags_source-5.8-5-source.hpkg") {
			Ok(o) => o,
			Err(e) => {
				println!("ERROR: {}", e);
				assert!(false);
				return;
			},
		};
		//println!("{}", hpkg);
	}
}