use std::path::{Path, PathBuf};
use std::fs::File;
use std::io::Write;
use std::mem::size_of;
use std::error::Error;
use flate2::write::ZlibEncoder;
use flate2::Compression;
use zstd::stream::write::Encoder as ZstdEncoder;
use crate::hpkg_common::*;
use crate::package::{PackageHeaderV2, HPKG_FILE_TYPE_FILE, HPKG_FILE_TYPE_DIRECTORY, HPKG_FILE_TYPE_SYMLINK};
const ATTR_DIRECTORY_ENTRY: u16 = 0;
const ATTR_FILE_TYPE: u16 = 1;
const ATTR_FILE_PERMISSIONS: u16 = 2;
const ATTR_FILE_USER: u16 = 3;
const ATTR_FILE_GROUP: u16 = 4;
const ATTR_FILE_MTIME: u16 = 6;
const ATTR_DATA: u16 = 13;
const ATTR_SYMLINK_PATH: u16 = 14;
const ATTR_PACKAGE_NAME: u16 = 15;
const ATTR_PACKAGE_SUMMARY: u16 = 16;
const ATTR_PACKAGE_DESCRIPTION: u16 = 17;
const ATTR_PACKAGE_VENDOR: u16 = 18;
const ATTR_PACKAGE_PACKAGER: u16 = 19;
const ATTR_PACKAGE_FLAGS: u16 = 20;
const ATTR_PACKAGE_ARCHITECTURE: u16 = 21;
const ATTR_PACKAGE_URL: u16 = 38;
const ATTR_PACKAGE_SOURCE_URL: u16 = 39;
const ATTR_PACKAGE_INSTALL_PATH: u16 = 40;
const ATTR_PACKAGE_BASE_PACKAGE: u16 = 41;
pub struct PackageWriter {
path: PathBuf,
compression: u16,
chunk_size: u32,
name: Option<String>,
summary: Option<String>,
description: Option<String>,
vendor: Option<String>,
packager: Option<String>,
flags: u32,
architecture: Option<u64>,
url: Option<String>,
source_url: Option<String>,
install_path: Option<String>,
base_package: Option<String>,
entries: Vec<FileEntryToWrite>,
}
#[derive(Debug, Clone)]
pub struct FileEntryToWrite {
pub path: String,
pub file_type: u32,
pub permissions: u32,
pub user: Option<String>,
pub group: Option<String>,
pub modified_time: Option<u64>,
pub symlink_target: Option<String>,
pub data: Vec<u8>,
}
impl FileEntryToWrite {
pub fn new_file(path: &str, data: &[u8]) -> Self {
FileEntryToWrite {
path: path.to_string(),
file_type: HPKG_FILE_TYPE_FILE,
permissions: 0o644,
user: None,
group: None,
modified_time: None,
symlink_target: None,
data: data.to_vec(),
}
}
pub fn new_directory(path: &str) -> Self {
FileEntryToWrite {
path: path.to_string(),
file_type: HPKG_FILE_TYPE_DIRECTORY,
permissions: 0o755,
user: None,
group: None,
modified_time: None,
symlink_target: None,
data: Vec::new(),
}
}
pub fn new_symlink(path: &str, target: &str) -> Self {
FileEntryToWrite {
path: path.to_string(),
file_type: HPKG_FILE_TYPE_SYMLINK,
permissions: 0o644,
user: None,
group: None,
modified_time: None,
symlink_target: Some(target.to_string()),
data: Vec::new(),
}
}
}
impl PackageWriter {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
PackageWriter {
path: path.as_ref().to_path_buf(),
compression: B_HPKG_COMPRESSION_NONE,
chunk_size: 65536,
name: None,
summary: None,
description: None,
vendor: None,
packager: None,
flags: 0,
architecture: None,
url: None,
source_url: None,
install_path: None,
base_package: None,
entries: Vec::new(),
}
}
pub fn set_name(&mut self, name: &str) -> &mut Self {
self.name = Some(name.to_string());
self
}
pub fn set_summary(&mut self, summary: &str) -> &mut Self {
self.summary = Some(summary.to_string());
self
}
pub fn set_description(&mut self, description: &str) -> &mut Self {
self.description = Some(description.to_string());
self
}
pub fn set_vendor(&mut self, vendor: &str) -> &mut Self {
self.vendor = Some(vendor.to_string());
self
}
pub fn set_packager(&mut self, packager: &str) -> &mut Self {
self.packager = Some(packager.to_string());
self
}
pub fn set_architecture(&mut self, arch: u64) -> &mut Self {
self.architecture = Some(arch);
self
}
pub fn set_flags(&mut self, flags: u32) -> &mut Self {
self.flags = flags;
self
}
pub fn set_url(&mut self, url: &str) -> &mut Self {
self.url = Some(url.to_string());
self
}
pub fn set_source_url(&mut self, url: &str) -> &mut Self {
self.source_url = Some(url.to_string());
self
}
pub fn set_install_path(&mut self, path: &str) -> &mut Self {
self.install_path = Some(path.to_string());
self
}
pub fn set_base_package(&mut self, base: &str) -> &mut Self {
self.base_package = Some(base.to_string());
self
}
pub fn set_compression(&mut self, compression: u16) -> &mut Self {
self.compression = compression;
self
}
pub fn add_entry(&mut self, entry: FileEntryToWrite) -> &mut Self {
self.entries.push(entry);
self
}
pub fn add_file(&mut self, path: &str, data: &[u8]) -> &mut Self {
self.entries.push(FileEntryToWrite::new_file(path, data));
self
}
pub fn add_directory(&mut self, path: &str) -> &mut Self {
self.entries.push(FileEntryToWrite::new_directory(path));
self
}
pub fn add_symlink(&mut self, path: &str, target: &str) -> &mut Self {
self.entries.push(FileEntryToWrite::new_symlink(path, target));
self
}
pub fn finish(&mut self) -> Result<(), Box<dyn Error>> {
let mut root = TocNode::new("");
for entry in &self.entries {
self.insert_entry_into_tree(&mut root, entry)?;
}
let mut data_pool: Vec<u8> = Vec::new();
self.assign_data_offsets(&mut root, &mut data_pool);
let toc_strings: Vec<u8> = Vec::new();
let mut toc_tree: Vec<u8> = Vec::new();
let toc_strings_count: u32 = 0;
self.serialize_toc_node(&mut root, &mut toc_tree)?;
let toc_strings_length = toc_strings.len() as u64;
let toc_length = (toc_strings_length + toc_tree.len() as u64) as u64;
let attr_strings: Vec<u8> = Vec::new();
let mut attr_tree: Vec<u8> = Vec::new();
self.serialize_package_attributes(&mut attr_tree)?;
let attr_strings_count: u32 = 0;
let attr_strings_length = attr_strings.len() as u32;
let attributes_length = (attr_strings_length + attr_tree.len() as u32) as u32;
let mut heap = Vec::new();
heap.extend_from_slice(&data_pool);
heap.extend_from_slice(&toc_strings);
heap.extend_from_slice(&toc_tree);
heap.extend_from_slice(&attr_strings);
heap.extend_from_slice(&attr_tree);
let heap_size_uncompressed = heap.len() as u64;
let chunk_count = (heap_size_uncompressed + self.chunk_size as u64 - 1) / self.chunk_size as u64;
let mut compressed_chunks: Vec<Vec<u8>> = Vec::new();
let mut chunk_size_table: Vec<u16> = Vec::new();
for i in 0..chunk_count {
let start = (i * self.chunk_size as u64) as usize;
let end = std::cmp::min(start + self.chunk_size as usize, heap.len());
let chunk_data = &heap[start..end];
let compressed = match self.compression {
B_HPKG_COMPRESSION_NONE => chunk_data.to_vec(),
B_HPKG_COMPRESSION_ZLIB => {
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(chunk_data)?;
let compressed = encoder.finish()?;
if compressed.len() < chunk_data.len() {
compressed
} else {
chunk_data.to_vec()
}
}
B_HPKG_COMPRESSION_ZSTD => {
let mut encoder = ZstdEncoder::new(Vec::new(), 3)?;
encoder.write_all(chunk_data)?;
let compressed = encoder.finish()?;
if compressed.len() < chunk_data.len() {
compressed
} else {
chunk_data.to_vec()
}
}
_ => return Err(From::from(format!("Unknown compression: {}", self.compression))),
};
compressed_chunks.push(compressed);
}
let mut heap_size_compressed: u64 = 0;
if self.compression == B_HPKG_COMPRESSION_NONE {
heap_size_compressed = heap_size_uncompressed;
} else {
for (i, chunk) in compressed_chunks.iter().enumerate() {
heap_size_compressed += chunk.len() as u64;
if i < compressed_chunks.len() - 1 {
let cookie = (chunk.len() as u16).wrapping_sub(1);
chunk_size_table.push(cookie);
}
}
heap_size_compressed += (chunk_size_table.len() * 2) as u64;
}
let header_size = size_of::<PackageHeaderV2>() as u16;
let header = PackageHeaderV2 {
magic: u32::from_ne_bytes([b'h', b'p', b'k', b'g']),
header_size: header_size.to_be(),
version: 2u16.to_be(),
total_size: (header_size as u64 + heap_size_compressed).to_be(),
minor_version: 0u16.to_be(),
heap_compression: self.compression.to_be(),
heap_chunk_size: self.chunk_size.to_be(),
heap_size_compressed: heap_size_compressed.to_be(),
heap_size_uncompressed: heap_size_uncompressed.to_be(),
attributes_length: attributes_length.to_be(),
attributes_strings_length: attr_strings_length.to_be(),
attributes_strings_count: attr_strings_count.to_be(),
reserved1: 0u32.to_be(),
toc_length: toc_length.to_be(),
toc_strings_length: toc_strings_length.to_be(),
toc_strings_count: (toc_strings_count as u64).to_be(),
};
let mut f = File::create(&self.path)?;
let header_bytes = unsafe {
std::slice::from_raw_parts(
&header as *const PackageHeaderV2 as *const u8,
size_of::<PackageHeaderV2>(),
)
};
f.write_all(header_bytes)?;
for chunk in &compressed_chunks {
f.write_all(chunk)?;
}
if self.compression != B_HPKG_COMPRESSION_NONE {
for &cookie in &chunk_size_table {
f.write_all(&cookie.to_be_bytes())?;
}
}
f.sync_all()?;
Ok(())
}
fn insert_entry_into_tree(&self, root: &mut TocNode, entry: &FileEntryToWrite) -> Result<(), Box<dyn Error>> {
let path = entry.path.trim_start_matches('/');
if path.is_empty() {
return Err(From::from("Empty entry path".to_string()));
}
let components: Vec<&str> = path.split('/').collect();
let mut node = root;
for (i, component) in components.iter().enumerate() {
let is_last = i == components.len() - 1;
if is_last {
node = node.get_or_create_child(component, entry);
} else {
node = node.get_or_create_dir_child(component);
}
}
Ok(())
}
fn assign_data_offsets(&self, node: &mut TocNode, pool: &mut Vec<u8>) {
if node.file_type == HPKG_FILE_TYPE_FILE && !node.data.is_empty() {
node.data_offset = Some(pool.len());
node.data_size = Some(node.data.len());
pool.extend_from_slice(&node.data);
}
for child in &mut node.children {
self.assign_data_offsets(child, pool);
}
}
fn serialize_toc_node(&self, node: &TocNode, buf: &mut Vec<u8>) -> Result<(), Box<dyn Error>> {
let has_children = !node.children.is_empty()
|| node.file_type != HPKG_FILE_TYPE_DIRECTORY;
write_attr_tag(buf, ATTR_DIRECTORY_ENTRY, HPKG_ATTR_TYPE_STRING, HPKG_ATTR_ENCODING_STRING_INLINE, has_children);
write_string_inline(buf, &node.name);
if has_children {
if node.file_type != HPKG_FILE_TYPE_DIRECTORY {
write_attr_tag(buf, ATTR_FILE_TYPE, HPKG_ATTR_TYPE_UINT, uint_encoding(node.file_type as u64), false);
write_int_value(buf, node.file_type as u64, uint_encoding(node.file_type as u64));
}
if node.permissions != 0o755 {
write_attr_tag(buf, ATTR_FILE_PERMISSIONS, HPKG_ATTR_TYPE_UINT, uint_encoding(node.permissions as u64), false);
write_int_value(buf, node.permissions as u64, uint_encoding(node.permissions as u64));
}
if let Some(ref user) = node.user {
write_attr_tag(buf, ATTR_FILE_USER, HPKG_ATTR_TYPE_STRING, HPKG_ATTR_ENCODING_STRING_INLINE, false);
write_string_inline(buf, user);
}
if let Some(ref group) = node.group {
write_attr_tag(buf, ATTR_FILE_GROUP, HPKG_ATTR_TYPE_STRING, HPKG_ATTR_ENCODING_STRING_INLINE, false);
write_string_inline(buf, group);
}
if let Some(time) = node.modified_time {
write_attr_tag(buf, ATTR_FILE_MTIME, HPKG_ATTR_TYPE_UINT, uint_encoding(time), false);
write_int_value(buf, time, uint_encoding(time));
}
if let Some(ref target) = node.symlink_path {
write_attr_tag(buf, ATTR_SYMLINK_PATH, HPKG_ATTR_TYPE_STRING, HPKG_ATTR_ENCODING_STRING_INLINE, false);
write_string_inline(buf, target);
} else if let (Some(offset), Some(size)) = (node.data_offset, node.data_size) {
if size > 0 {
if size > 8 {
write_attr_tag(buf, ATTR_DATA, HPKG_ATTR_TYPE_RAW, HPKG_ATTR_ENCODING_RAW_HEAP, false);
write_unsigned_leb128(buf, size as u64);
write_unsigned_leb128(buf, offset as u64);
} else {
write_attr_tag(buf, ATTR_DATA, HPKG_ATTR_TYPE_RAW, HPKG_ATTR_ENCODING_RAW_INLINE, false);
write_unsigned_leb128(buf, size as u64);
if let Some(ref data) = node.data.as_slice().get(..size) {
buf.extend_from_slice(data);
}
}
}
}
for child in &node.children {
self.serialize_toc_node(child, buf)?;
}
write_unsigned_leb128(buf, 0);
}
Ok(())
}
fn serialize_package_attributes(&self, buf: &mut Vec<u8>) -> Result<(), Box<dyn Error>> {
let pairs: Vec<(u16, &str)> = vec![
(ATTR_PACKAGE_NAME, "name"),
(ATTR_PACKAGE_SUMMARY, "summary"),
(ATTR_PACKAGE_DESCRIPTION, "description"),
(ATTR_PACKAGE_VENDOR, "vendor"),
(ATTR_PACKAGE_PACKAGER, "packager"),
(ATTR_PACKAGE_URL, "url"),
(ATTR_PACKAGE_SOURCE_URL, "source_url"),
(ATTR_PACKAGE_INSTALL_PATH, "install_path"),
(ATTR_PACKAGE_BASE_PACKAGE, "base_package"),
];
for (attr_id, field) in pairs {
let value: Option<&str> = match field {
"name" => self.name.as_deref(),
"summary" => self.summary.as_deref(),
"description" => self.description.as_deref(),
"vendor" => self.vendor.as_deref(),
"packager" => self.packager.as_deref(),
"url" => self.url.as_deref(),
"source_url" => self.source_url.as_deref(),
"install_path" => self.install_path.as_deref(),
"base_package" => self.base_package.as_deref(),
_ => None,
};
if let Some(v) = value {
write_attr_tag(buf, attr_id, HPKG_ATTR_TYPE_STRING, HPKG_ATTR_ENCODING_STRING_INLINE, false);
write_string_inline(buf, v);
}
}
if self.flags != 0 {
write_attr_tag(buf, ATTR_PACKAGE_FLAGS, HPKG_ATTR_TYPE_UINT, uint_encoding(self.flags as u64), false);
write_int_value(buf, self.flags as u64, uint_encoding(self.flags as u64));
}
if let Some(arch) = self.architecture {
write_attr_tag(buf, ATTR_PACKAGE_ARCHITECTURE, HPKG_ATTR_TYPE_UINT, uint_encoding(arch), false);
write_int_value(buf, arch, uint_encoding(arch));
}
Ok(())
}
}
struct TocNode {
name: String,
file_type: u32,
permissions: u32,
user: Option<String>,
group: Option<String>,
modified_time: Option<u64>,
symlink_path: Option<String>,
data: Vec<u8>,
data_offset: Option<usize>,
data_size: Option<usize>,
children: Vec<TocNode>,
}
impl TocNode {
fn new(name: &str) -> Self {
TocNode {
name: name.to_string(),
file_type: HPKG_FILE_TYPE_DIRECTORY,
permissions: 0o755,
user: None,
group: None,
modified_time: None,
symlink_path: None,
data: Vec::new(),
data_offset: None,
data_size: None,
children: Vec::new(),
}
}
fn get_or_create_dir_child(&mut self, name: &str) -> &mut TocNode {
let idx = self.children.iter().position(|c| c.name == name);
let idx = idx.unwrap_or_else(|| {
self.children.push(TocNode::new(name));
self.children.len() - 1
});
&mut self.children[idx]
}
fn get_or_create_child(&mut self, name: &str, entry: &FileEntryToWrite) -> &mut TocNode {
let idx = self.children.iter().position(|c| c.name == name);
let idx = idx.unwrap_or_else(|| {
self.children.push(TocNode {
name: name.to_string(),
file_type: entry.file_type,
permissions: entry.permissions,
user: entry.user.clone(),
group: entry.group.clone(),
modified_time: entry.modified_time,
symlink_path: entry.symlink_target.clone(),
data: entry.data.clone(),
data_offset: None,
data_size: None,
children: Vec::new(),
});
self.children.len() - 1
});
&mut self.children[idx]
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::package::Package;
#[test]
fn test_writer_round_trip_uncompressed() {
let tmp = std::env::temp_dir().join("test_round_trip_uncompressed.hpkg");
let _ = std::fs::remove_file(&tmp);
{
let mut writer = PackageWriter::new(&tmp);
writer.set_name("test_pkg")
.set_summary("A test package")
.set_vendor("Test Vendor")
.set_architecture(ARCH_X86_64)
.set_compression(B_HPKG_COMPRESSION_NONE)
.set_url("https://example.com");
writer.add_file("bin/hello", b"#!/bin/sh\necho hello\n");
writer.add_file("data/file.txt", b"Hello, HPKG World!\n");
writer.add_directory("data/subdir");
writer.add_symlink("bin/link", "hello");
writer.finish().expect("write should succeed");
}
let pkg = Package::load(&tmp).expect("should load written package");
assert_eq!(pkg.name.as_deref(), Some("test_pkg"));
assert_eq!(pkg.summary.as_deref(), Some("A test package"));
assert_eq!(pkg.vendor.as_deref(), Some("Test Vendor"));
assert_eq!(pkg.architecture.as_deref(), Some("x86_64"));
assert_eq!(pkg.url.as_deref(), Some("https://example.com"));
let files = pkg.list_files();
let file_names: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
assert!(file_names.contains(&"bin/hello"), "should contain bin/hello, got {:?}", file_names);
assert!(file_names.contains(&"data/file.txt"), "should contain data/file.txt");
let hello_data = pkg.read_file("bin/hello").expect("should read bin/hello");
assert_eq!(hello_data, b"#!/bin/sh\necho hello\n");
let txt_data = pkg.read_file("data/file.txt").expect("should read data/file.txt");
assert_eq!(txt_data, b"Hello, HPKG World!\n");
let _ = std::fs::remove_file(&tmp);
}
#[test]
fn test_writer_round_trip_zstd() {
let tmp = std::env::temp_dir().join("test_round_trip_zstd.hpkg");
let _ = std::fs::remove_file(&tmp);
{
let mut writer = PackageWriter::new(&tmp);
writer.set_name("zstd_pkg")
.set_summary("Zstd compressed test")
.set_vendor("Test")
.set_architecture(ARCH_ANY)
.set_compression(B_HPKG_COMPRESSION_ZSTD);
writer.add_file("data.txt", b"Zstd compression test data.\n");
writer.finish().expect("write should succeed");
}
let pkg = Package::load(&tmp).expect("should load zstd-compressed package");
assert_eq!(pkg.name.as_deref(), Some("zstd_pkg"));
assert!(pkg.header.unwrap().heap_compression == B_HPKG_COMPRESSION_ZSTD);
let data = pkg.read_file("data.txt").expect("should read data.txt");
assert_eq!(data, b"Zstd compression test data.\n");
let _ = std::fs::remove_file(&tmp);
}
#[test]
fn test_writer_round_trip_zlib() {
let tmp = std::env::temp_dir().join("test_round_trip_zlib.hpkg");
let _ = std::fs::remove_file(&tmp);
{
let mut writer = PackageWriter::new(&tmp);
writer.set_name("zlib_pkg")
.set_summary("Zlib compressed test")
.set_vendor("Test")
.set_architecture(ARCH_SOURCE)
.set_compression(B_HPKG_COMPRESSION_ZLIB);
writer.add_file("readme.txt", b"This is a test file.\n");
writer.finish().expect("write should succeed");
}
let pkg = Package::load(&tmp).expect("should load zlib-compressed package");
assert_eq!(pkg.name.as_deref(), Some("zlib_pkg"));
assert!(pkg.header.unwrap().heap_compression == B_HPKG_COMPRESSION_ZLIB);
let data = pkg.read_file("readme.txt").expect("should read readme.txt");
assert_eq!(data, b"This is a test file.\n");
let _ = std::fs::remove_file(&tmp);
}
}