use std::error;
use std::fmt;
use std::fs::File;
use std::io::{BufReader, Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use flate2::read::ZlibDecoder;
use zstd;
use crate::hpkg_common::*;
#[derive(Debug, Clone)]
#[repr(C)]
pub struct RepositoryHeaderV2 {
pub magic: u32,
pub header_size: u16,
pub version: u16,
pub total_size: u64,
pub minor_version: u16,
pub heap_compression: u16,
pub heap_chunk_size: u32,
pub heap_size_compressed: u64,
pub heap_size_uncompressed: u64,
pub info_length: u32,
pub reserved1: u32,
pub package_length: u64,
pub package_strings_length: u64,
pub package_strings_count: u64,
}
#[derive(Debug, Clone)]
pub struct RepositoryInfo {
pub name: Option<String>,
pub identifier: Option<String>,
pub base_url: Option<String>,
pub vendor: Option<String>,
pub summary: Option<String>,
pub priority: Option<u8>,
pub architecture: Option<String>,
pub license_names: Vec<String>,
pub license_texts: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct PackageInfo {
pub name: Option<String>,
pub summary: Option<String>,
pub description: Option<String>,
pub vendor: Option<String>,
pub packager: Option<String>,
pub flags: u32,
pub architecture: Option<String>,
pub checksum: Option<String>,
pub url: Option<String>,
pub source_url: Option<String>,
pub install_path: Option<String>,
pub version_major: Option<String>,
pub version_minor: Option<String>,
pub version_micro: Option<String>,
pub version_revision: Option<u64>,
pub copyrights: Vec<String>,
pub licenses: Vec<String>,
pub provides: Vec<String>,
pub requires: Vec<String>,
}
pub struct Repository {
pub filename: Option<PathBuf>,
pub header: Option<RepositoryHeaderV2>,
pub info: RepositoryInfo,
pub packages: Vec<PackageInfo>,
heap_chunk_offsets: Vec<u64>,
flattened_heap: Vec<u8>,
}
impl RepositoryInfo {
pub fn new() -> RepositoryInfo {
RepositoryInfo {
name: None,
identifier: None,
base_url: None,
vendor: None,
summary: None,
priority: None,
architecture: None,
license_names: Vec::new(),
license_texts: Vec::new(),
}
}
}
impl PackageInfo {
pub fn new() -> PackageInfo {
PackageInfo {
name: None,
summary: None,
description: None,
vendor: None,
packager: None,
flags: 0,
architecture: None,
checksum: None,
url: None,
source_url: None,
install_path: None,
version_major: None,
version_minor: None,
version_micro: None,
version_revision: None,
copyrights: Vec::new(),
licenses: Vec::new(),
provides: Vec::new(),
requires: Vec::new(),
}
}
}
fn parse_header<P: AsRef<Path>>(repo_file: P) -> Result<RepositoryHeaderV2, Box<dyn error::Error>> {
let mut f = File::open(repo_file.as_ref())?;
f.seek(SeekFrom::Start(0))?;
let reader = BufReader::new(f);
let mut header = read_struct::<RepositoryHeaderV2, _>(reader)?;
let magic_bytes = header.magic.to_ne_bytes();
if magic_bytes != [b'h', b'p', b'k', b'r'] {
return Err(From::from(format!("Unknown magic: {:?}", magic_bytes)));
}
header.header_size = u16::from_be(header.header_size);
header.version = u16::from_be(header.version);
header.total_size = u64::from_be(header.total_size);
header.minor_version = u16::from_be(header.minor_version);
header.heap_compression = u16::from_be(header.heap_compression);
header.heap_chunk_size = u32::from_be(header.heap_chunk_size);
header.heap_size_compressed = u64::from_be(header.heap_size_compressed);
header.heap_size_uncompressed = u64::from_be(header.heap_size_uncompressed);
header.info_length = u32::from_be(header.info_length);
header.reserved1 = u32::from_be(header.reserved1);
header.package_length = u64::from_be(header.package_length);
header.package_strings_length = u64::from_be(header.package_strings_length);
header.package_strings_count = u64::from_be(header.package_strings_count);
if header.version != 2 {
return Err(From::from(format!("Unknown repo version: {}", header.version)));
}
if header.header_size as u64 + header.heap_size_compressed != header.total_size {
return Err(From::from(format!("Invalid repo header lengths")));
}
Ok(header)
}
impl Repository {
fn heap_chunk_count(&self) -> Result<u64, Box<dyn error::Error>> {
let header = self.header.as_ref().ok_or("No header loaded")?;
let chunk_size = header.heap_chunk_size as u64;
Ok((header.heap_size_uncompressed + chunk_size - 1) / chunk_size)
}
fn heap_chunkify(&mut self) -> Result<u64, Box<dyn error::Error>> {
let chunks = self.heap_chunk_count()?;
let (heap_compression, heap_chunk_size, header_size, heap_size_compressed) = {
let h = self.header.as_ref().unwrap();
(h.heap_compression, h.heap_chunk_size, h.header_size, h.heap_size_compressed)
};
self.heap_chunk_offsets.push(0);
if heap_compression == 0 {
for i in 1..chunks {
self.heap_chunk_offsets.push(i as u64 * heap_chunk_size as u64);
}
} else {
let filename = self.filename.as_ref().unwrap().clone();
let chunk_size_table_len = (chunks - 1) * 2;
if heap_size_compressed <= chunk_size_table_len {
return Err(From::from(format!(
"Compressed heap smaller than chunk size table"
)));
}
let table_start = header_size as u64 + heap_size_compressed - chunk_size_table_len;
let mut f = File::open(&filename)?;
f.seek(SeekFrom::Start(table_start))?;
let mut chunkbuffer = vec![0; chunk_size_table_len as usize];
BufReader::new(&f).read_exact(&mut chunkbuffer)?;
for chunk_index in 0..chunkbuffer.len() / 2 {
let base = chunk_index * 2;
let mut raw_cookies: u64 = ((chunkbuffer[base] as u64) << 8)
| chunkbuffer[base + 1] as u64;
raw_cookies += self.heap_chunk_offsets.last().unwrap() + 1;
self.heap_chunk_offsets.push(raw_cookies);
}
}
Ok(0)
}
}
impl Repository {
fn read_string_from(&self, offset: &mut usize) -> Result<&str, Box<dyn error::Error>> {
let data = &self.flattened_heap;
let start = *offset;
while *offset < data.len() && data[*offset] != 0 {
*offset += 1;
}
if *offset >= data.len() {
return Err(From::from(
"Unexpected end of heap data in string table".to_string(),
));
}
let s = std::str::from_utf8(&data[start..*offset])?;
*offset += 1;
Ok(s)
}
fn parse_string_table(&self, offset: usize, count: u64) -> Result<Vec<String>, Box<dyn error::Error>> {
let mut pos = offset;
let mut table = Vec::with_capacity(count as usize);
for _ in 0..count {
let s = self.read_string_from(&mut pos)?.to_string();
table.push(s);
}
Ok(table)
}
fn read_attr_value(
&self,
offset: &mut usize,
type_: u16,
encoding: u16,
string_table: &[String],
) -> Result<AttrValue, Box<dyn error::Error>> {
match type_ {
HPKG_ATTR_TYPE_INT | HPKG_ATTR_TYPE_UINT => {
let v: u64 = match encoding {
0 => {
let b = self.flattened_heap[*offset];
*offset += 1;
b as u64
}
1 => {
let v = u16::from_be_bytes(
self.flattened_heap[*offset..*offset + 2].try_into().unwrap(),
);
*offset += 2;
v as u64
}
2 => {
let v = u32::from_be_bytes(
self.flattened_heap[*offset..*offset + 4].try_into().unwrap(),
);
*offset += 4;
v as u64
}
3 => {
let v = u64::from_be_bytes(
self.flattened_heap[*offset..*offset + 8].try_into().unwrap(),
);
*offset += 8;
v
}
_ => return Err(From::from(format!("Unknown int encoding {}", encoding))),
};
if type_ == HPKG_ATTR_TYPE_INT {
Ok(AttrValue::Int(v as i64))
} else {
Ok(AttrValue::Uint(v))
}
}
HPKG_ATTR_TYPE_STRING => {
let s = if encoding == 0 {
self.read_string_from(offset)?.to_string()
} else {
let idx = read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
if idx >= string_table.len() {
return Err(From::from(format!(
"String table index {} out of bounds",
idx
)));
}
string_table[idx].clone()
};
Ok(AttrValue::String(s))
}
HPKG_ATTR_TYPE_RAW => {
let size = read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
if encoding == 0 {
let data = self.flattened_heap[*offset..*offset + size].to_vec();
*offset += size;
Ok(AttrValue::Raw(data))
} else {
let heap_offset =
read_unsigned_leb128(&self.flattened_heap, offset)? as usize;
let data =
self.flattened_heap[heap_offset..heap_offset + size].to_vec();
Ok(AttrValue::Raw(data))
}
}
_ => Err(From::from(format!("Unknown attribute type {}", type_))),
}
}
fn skip_attribute_value(&self, offset: &mut usize, type_: u16, encoding: u16) {
match type_ {
HPKG_ATTR_TYPE_INT | HPKG_ATTR_TYPE_UINT => match encoding {
0 => *offset += 1,
1 => *offset += 2,
2 => *offset += 4,
3 => *offset += 8,
_ => {}
},
HPKG_ATTR_TYPE_STRING => {
if encoding == 0 {
while *offset < self.flattened_heap.len()
&& self.flattened_heap[*offset] != 0
{
*offset += 1;
}
if *offset < self.flattened_heap.len() {
*offset += 1;
}
} else {
let _ = read_unsigned_leb128(&self.flattened_heap, offset);
}
}
HPKG_ATTR_TYPE_RAW => {
if let Ok(size) = read_unsigned_leb128(&self.flattened_heap, offset) {
if encoding == 0 {
*offset += size as usize;
} else {
let _ = read_unsigned_leb128(&self.flattened_heap, offset);
}
}
}
_ => {}
}
}
fn skip_attribute_tree(&self, offset: &mut usize) {
let mut depth = 0usize;
loop {
if *offset >= self.flattened_heap.len() {
return;
}
let Ok(tag_raw) = read_unsigned_leb128(&self.flattened_heap, offset) else {
return;
};
if tag_raw == 0 {
if depth == 0 {
return;
}
depth -= 1;
continue;
}
let (_id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);
self.skip_attribute_value(offset, type_, encoding);
if has_children {
depth += 1;
}
}
}
}
impl Repository {
fn parse_repository_info_section(&mut self) -> Result<(), Box<dyn error::Error>> {
let length = match &self.header {
Some(h) => h.info_length as usize,
None => return Err(From::from("No header loaded")),
};
if length == 0 {
return Ok(());
}
let data = &self.flattened_heap[..length];
let fields: &[(&str, &str)] = &[
("name", "string"),
("identifier", "string"),
("baseurl", "string"),
("vendor", "string"),
("summary", "string"),
("licenseName", "string"),
("licenseText", "string"),
("priority", "int"),
("architecture", "int"),
];
for &(field_name, field_type) in fields {
let name_bytes = field_name.as_bytes();
let mut search_pos = 0;
while search_pos + name_bytes.len() + 1 < data.len() {
if data[search_pos..search_pos + name_bytes.len()] == *name_bytes
&& data[search_pos + name_bytes.len()] == 0
{
let value_pos = search_pos + name_bytes.len() + 1;
match field_type {
"string" => {
if value_pos + 4 <= data.len() {
let str_len = u32::from_le_bytes(
data[value_pos..value_pos + 4].try_into().unwrap(),
) as usize;
if value_pos + 4 + str_len <= data.len() && str_len > 0 {
let s = String::from_utf8_lossy(
&data[value_pos + 4..value_pos + 4 + str_len - 1],
)
.to_string();
match field_name {
"name" => self.info.name = Some(s),
"identifier" => self.info.identifier = Some(s),
"baseurl" => self.info.base_url = Some(s),
"vendor" => self.info.vendor = Some(s),
"summary" => self.info.summary = Some(s),
"licenseName" => self.info.license_names.push(s),
"licenseText" => self.info.license_texts.push(s),
_ => {}
}
}
}
}
"int" => {
if value_pos + 1 <= data.len() {
let v = data[value_pos] as u8;
match field_name {
"priority" => self.info.priority = Some(v),
"architecture" => {
self.info.architecture = Some(arch_to_string(v as u64))
}
_ => {}
}
}
}
_ => {}
}
break;
}
search_pos += 1;
}
}
Ok(())
}
}
impl Repository {
fn parse_packages_section(&mut self) -> Result<(), Box<dyn error::Error>> {
let header = match &self.header {
Some(h) => h.clone(),
None => return Err(From::from("No header loaded")),
};
if header.package_length == 0 {
return Ok(());
}
let heap_size = header.heap_size_uncompressed as usize;
let info_len = header.info_length as usize;
if info_len + header.package_length as usize > heap_size {
return Err(From::from(format!(
"Package attributes section ({} @ {}) exceeds heap size ({})",
header.package_length, info_len, heap_size
)));
}
let strings_len = header.package_strings_length as usize;
let strings_count = header.package_strings_count;
let section_start = info_len;
let strings_offset = section_start;
let main_offset = section_start + strings_len;
let string_table = if strings_count > 0 {
self.parse_string_table(strings_offset, strings_count)?
} else {
Vec::new()
};
let mut pos = main_offset;
let end = section_start + header.package_length as usize;
self.parse_package_list(&mut pos, &string_table, end)?;
Ok(())
}
fn parse_package_list(
&mut self,
offset: &mut usize,
string_table: &[String],
end_bound: usize,
) -> Result<(), Box<dyn error::Error>> {
loop {
if *offset >= end_bound || *offset >= self.flattened_heap.len() {
return Ok(());
}
let tag_raw = read_unsigned_leb128(&self.flattened_heap, offset)?;
if tag_raw == 0 {
return Ok(());
}
let (id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);
if id != 54 {
self.read_attr_value(offset, type_, encoding, string_table)?;
if has_children {
self.skip_attribute_tree(offset);
}
continue;
}
let pkg_name = match self.read_attr_value(offset, type_, encoding, string_table)? {
AttrValue::String(s) => s,
_ => String::new(),
};
let mut pkg = PackageInfo::new();
pkg.name = Some(pkg_name);
if has_children {
self.parse_package_attributes(offset, string_table, &mut pkg)?;
}
self.packages.push(pkg);
}
}
fn parse_package_attributes(
&self,
offset: &mut usize,
string_table: &[String],
pkg: &mut PackageInfo,
) -> Result<(), Box<dyn error::Error>> {
loop {
if *offset >= self.flattened_heap.len() {
return Ok(());
}
let tag_raw = read_unsigned_leb128(&self.flattened_heap, offset)?;
if tag_raw == 0 {
return Ok(());
}
let (id, type_, encoding, has_children) = decode_attribute_tag(tag_raw);
let value = self.read_attr_value(offset, type_, encoding, string_table)?;
match id {
15 => {
if let AttrValue::String(s) = &value {
pkg.name = Some(s.clone());
}
}
16 => {
if let AttrValue::String(s) = &value {
pkg.summary = Some(s.clone());
}
}
17 => {
if let AttrValue::String(s) = &value {
pkg.description = Some(s.clone());
}
}
18 => {
if let AttrValue::String(s) = &value {
pkg.vendor = Some(s.clone());
}
}
19 => {
if let AttrValue::String(s) = &value {
pkg.packager = Some(s.clone());
}
}
20 => {
match &value {
AttrValue::Uint(v) => pkg.flags = *v as u32,
AttrValue::Int(v) => pkg.flags = *v as u32,
_ => {}
}
}
21 => {
match &value {
AttrValue::Uint(v) => pkg.architecture = Some(arch_to_string(*v)),
AttrValue::Int(v) => pkg.architecture = Some(arch_to_string(*v as u64)),
_ => {}
}
}
22 => {
if let AttrValue::String(s) = &value {
pkg.version_major = Some(s.clone());
}
}
23 => {
if let AttrValue::String(s) = &value {
pkg.version_minor = Some(s.clone());
}
}
24 => {
if let AttrValue::String(s) = &value {
pkg.version_micro = Some(s.clone());
}
}
25 => {
match &value {
AttrValue::Uint(v) => pkg.version_revision = Some(*v),
AttrValue::Int(v) => pkg.version_revision = Some(*v as u64),
_ => {}
}
}
26 => {
if let AttrValue::String(s) = &value {
pkg.copyrights.push(s.clone());
}
}
27 => {
if let AttrValue::String(s) = &value {
pkg.licenses.push(s.clone());
}
}
28 => {
if let AttrValue::String(s) = &value {
pkg.provides.push(s.clone());
}
}
29 => {
if let AttrValue::String(s) = &value {
pkg.requires.push(s.clone());
}
}
35 => {
if let AttrValue::String(s) = &value {
pkg.checksum = Some(s.clone());
}
}
38 => {
if let AttrValue::String(s) = &value {
pkg.url = Some(s.clone());
}
}
39 => {
if let AttrValue::String(s) = &value {
pkg.source_url = Some(s.clone());
}
}
40 => {
if let AttrValue::String(s) = &value {
pkg.install_path = Some(s.clone());
}
}
_ => {}
}
if has_children {
self.skip_attribute_tree(offset);
}
}
}
}
impl fmt::Display for Repository {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let name = self.info.name.as_deref().unwrap_or("(unknown)");
let vendor = self.info.vendor.as_deref().unwrap_or("(unknown)");
let summary = self.info.summary.as_deref().unwrap_or("(unknown)");
let arch = self.info.architecture.as_deref().unwrap_or("(unknown)");
write!(
f,
"repository. Name {:?}, Vendor {:?}, Summary {:?}, Arch {:?}",
name, vendor, summary, arch
)
}
}
impl fmt::Debug for Repository {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
writeln!(f, "Haiku Repository")?;
if let Some(ref h) = self.header {
writeln!(f, " header: {} v{}.{}", h.total_size, h.version, h.minor_version)?;
writeln!(f, " compression: {}", h.heap_compression)?;
writeln!(f, " heap: {} -> {} uncompressed",
h.heap_size_compressed, h.heap_size_uncompressed)?;
writeln!(f, " info_length: {}", h.info_length)?;
writeln!(f, " packages: {} entries ({} strings)",
h.package_length, h.package_strings_count)?;
}
writeln!(f, " name: {:?}", self.info.name)?;
writeln!(f, " vendor: {:?}", self.info.vendor)?;
writeln!(f, " summary: {:?}", self.info.summary)?;
writeln!(f, " architecture: {:?}", self.info.architecture)?;
writeln!(f, " packages count: {}", self.packages.len())?;
for pkg in self.packages.iter().take(5) {
writeln!(f, " - {}", pkg.name.as_deref().unwrap_or("(unnamed)"))?;
}
if self.packages.len() > 5 {
writeln!(f, " ... and {} more", self.packages.len() - 5)?;
}
Ok(())
}
}
impl Repository {
pub fn new() -> Repository {
Repository {
filename: None,
header: None,
info: RepositoryInfo::new(),
packages: Vec::new(),
heap_chunk_offsets: Vec::new(),
flattened_heap: Vec::new(),
}
}
pub fn load<P: AsRef<Path>>(repo_file: P) -> Result<Repository, Box<dyn error::Error>> {
let mut repo = Repository::new();
repo.filename = Some(repo_file.as_ref().to_path_buf());
let header = self::parse_header(repo_file)?;
if header.header_size as u64 + header.heap_size_compressed != header.total_size {
return Err(From::from(format!("Invalid repo file: header + heap != total_size")));
}
repo.header = Some(header);
repo.heap_chunkify()?;
{
let chunks = repo.heap_chunk_count()?;
let filename = repo.filename.as_ref().unwrap().clone();
let header = repo.header.as_ref().unwrap();
let heap_size = header.heap_size_uncompressed as usize;
let mut flat = vec![0u8; heap_size];
let mut dest_offset = 0usize;
let chunk_count = chunks as usize;
for chunk_index in 0..chunk_count {
let chunk_offset = header.header_size as usize
+ if header.heap_compression == 0 {
(chunk_index as u64 * header.heap_chunk_size as u64) as usize
} else {
repo.heap_chunk_offsets[chunk_index] as usize
};
let remaining = heap_size - dest_offset;
let chunk_len = remaining.min(header.heap_chunk_size as usize);
let mut f = File::open(&filename)?;
f.seek(SeekFrom::Start(chunk_offset as u64))?;
if header.heap_compression == 0 {
f.read_exact(&mut flat[dest_offset..dest_offset + chunk_len])?;
dest_offset += chunk_len;
} else {
let compressed_size: usize = if chunk_index + 1 < chunk_count {
(repo.heap_chunk_offsets[chunk_index + 1]
- repo.heap_chunk_offsets[chunk_index]) as usize
} else {
let chunk_table_len = (chunk_count as u64 - 1) * 2;
let total_compressed = header.heap_size_compressed - chunk_table_len;
(total_compressed
- repo.heap_chunk_offsets[chunk_index]) as usize
};
let mut compressed = vec![0u8; compressed_size];
f.read_exact(&mut compressed)?;
if compressed_size < chunk_len {
let mut reader: Box<dyn Read> = match header.heap_compression {
B_HPKG_COMPRESSION_ZLIB => {
Box::new(ZlibDecoder::new(&compressed[..]))
}
B_HPKG_COMPRESSION_ZSTD => Box::new(
zstd::stream::read::Decoder::new(&compressed[..])?,
),
_ => {
return Err(From::from(format!(
"Unknown repo heap compression: {}",
header.heap_compression
)))
}
};
reader.read_exact(&mut flat[dest_offset..dest_offset + chunk_len])?;
} else {
flat[dest_offset..dest_offset + chunk_len]
.copy_from_slice(&compressed[..chunk_len]);
}
dest_offset += chunk_len;
}
}
repo.flattened_heap = flat;
}
repo.parse_repository_info_section()?;
repo.parse_packages_section()?;
Ok(repo)
}
pub fn repository_info(&self) -> &RepositoryInfo {
&self.info
}
pub fn packages(&self) -> &[PackageInfo] {
&self.packages
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_new_repository() {
let repo = Repository::new();
assert!(repo.header.is_none());
assert!(repo.info.name.is_none());
}
#[test]
fn test_load_valid_repository() {
let repo = match Repository::load("sample/repo") {
Ok(o) => o,
Err(e) => {
println!("ERROR: {}", e);
assert!(false);
return;
}
};
assert!(repo.header.is_some());
}
#[test]
fn test_load_invalid_repository() {
assert!(Repository::load("sample/not-repo").is_err());
}
#[test]
fn test_total_size() {
let metadata = match std::fs::metadata("sample/repo") {
Ok(o) => o,
Err(e) => {
println!("ERROR: {}", e);
assert!(false);
return;
}
};
let hpkr = match Repository::load("sample/repo") {
Ok(o) => o,
Err(e) => {
println!("ERROR: {}", e);
assert!(false);
return;
}
};
let header = match hpkr.header {
Some(o) => o,
None => {
println!("ERROR: Invalid Header!");
assert!(false);
return;
}
};
assert_eq!(metadata.len(), header.total_size);
}
#[test]
fn test_repository_info() {
let repo = match Repository::load("sample/repo") {
Ok(o) => o,
Err(e) => {
println!("ERROR: {}", e);
assert!(false);
return;
}
};
println!("Repository Info: {:?}", repo.info);
println!("{:?}", repo);
assert!(
repo.info.name.is_some() || repo.info.vendor.is_some(),
"Expected at least some repository metadata, got {:?}",
repo.info
);
}
#[test]
fn test_repository_packages() {
let repo = match Repository::load("sample/repo") {
Ok(o) => o,
Err(e) => {
println!("ERROR: {}", e);
assert!(false);
return;
}
};
println!("Found {} packages in repository", repo.packages.len());
for pkg in &repo.packages {
println!(
" {} (vendor={:?}, arch={:?})",
pkg.name.as_deref().unwrap_or("(unnamed)"),
pkg.vendor,
pkg.architecture
);
}
}
}