use std::collections::HashMap;
use std::io;
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock, RwLock};
use crate::dom::Stylesheet;
use crate::import::{ChapterId, Importer, SpineEntry, resolve_path_based_href};
use crate::io::{ByteSource, FileSource};
use crate::mobi::parser::{
DivElement, SkeletonFile, parse_div_index, parse_ncx_index, parse_skel_index, read_index,
};
use crate::mobi::{
Compression, Encoding, HuffCdicReader, MobiFormat, MobiHeader, NULL_INDEX, PdbInfo, TocNode,
build_toc_from_ncx, decode_font_record, detect_font_type, detect_image_type,
is_metadata_record, palmdoc, parse_exth, parse_fdst, strip_trailing_data, transform,
};
use crate::model::{AnchorTarget, Chapter, GlobalNodeId, Landmark, Metadata, TocEntry};
pub struct Azw3Importer {
source: Arc<dyn ByteSource>,
pdb: PdbInfo,
mobi: MobiHeader,
record_offset: usize,
file_len: u64,
metadata: Metadata,
toc: Vec<TocEntry>,
landmarks: Vec<Landmark>,
spine: Vec<SpineEntry>,
chapter_paths: Vec<String>,
kf8: Kf8Structure,
text_cache: OnceLock<Vec<u8>>,
parts_cache: OnceLock<Vec<(String, Vec<u8>)>>,
text_init: Mutex<()>,
chapter_cache: RwLock<HashMap<u32, Vec<u8>>>,
assets: Vec<String>,
css_cache: RwLock<HashMap<String, Arc<Stylesheet>>>,
element_id_map: RwLock<HashMap<String, GlobalNodeId>>,
toc_positions: HashMap<usize, TocPosition>,
}
#[derive(Debug, Clone, Copy)]
struct TocPosition {
byte_pos: u32,
file_num: u32,
}
struct Kf8Structure {
flow_table: Vec<(usize, usize)>,
files: Vec<SkeletonFile>,
elems: Vec<DivElement>,
}
impl Importer for Azw3Importer {
fn open(path: &Path) -> crate::Result<Self> {
let file = std::fs::File::open(path)?;
let source = Arc::new(FileSource::new(file)?);
Self::from_source(source)
}
fn metadata(&self) -> &Metadata {
&self.metadata
}
fn toc(&self) -> &[TocEntry] {
&self.toc
}
fn landmarks(&self) -> &[Landmark] {
&self.landmarks
}
fn spine(&self) -> &[SpineEntry] {
&self.spine
}
fn source_id(&self, id: ChapterId) -> Option<&str> {
self.chapter_paths.get(id.0 as usize).map(|s| s.as_str())
}
fn load_raw(&self, id: ChapterId) -> crate::Result<Vec<u8>> {
if let Ok(cache) = self.chapter_cache.read()
&& let Some(content) = cache.get(&id.0)
{
return Ok(content.clone());
}
let content = self.build_chapter(id.0, self.cached_text()?)?;
if let Ok(mut cache) = self.chapter_cache.write() {
cache.insert(id.0, content.clone());
}
Ok(content)
}
fn list_assets(&self) -> &[String] {
&self.assets
}
fn load_asset(&self, path: &str) -> crate::Result<Vec<u8>> {
let idx: usize = path
.strip_prefix("images/image_")
.or_else(|| path.strip_prefix("fonts/font_"))
.and_then(|s| s.split('.').next())
.and_then(|s| s.parse().ok())
.ok_or_else(|| crate::Error::NotFound {
what: format!("asset {}", path),
})?;
Ok(self.load_image_record(idx)?)
}
fn load_stylesheet(&self, path: &str) -> Option<Arc<Stylesheet>> {
if let Ok(cache) = self.css_cache.read()
&& let Some(sheet) = cache.get(path)
{
return Some(Arc::clone(sheet));
}
let css_bytes = self.load_asset(path).ok()?;
let css_str = String::from_utf8_lossy(&css_bytes);
let sheet = Arc::new(Stylesheet::parse(&css_str));
match self.css_cache.write() {
Ok(mut cache) => Some(Arc::clone(cache.entry(path.to_string()).or_insert(sheet))),
Err(_) => Some(sheet),
}
}
fn index_anchors(&self, chapters: &[(ChapterId, Arc<Chapter>)]) {
let mut element_id_map = HashMap::new();
for (chapter_id, chapter) in chapters {
let chapter_path = match self.chapter_paths.get(chapter_id.0 as usize) {
Some(p) => p.as_str(),
None => continue,
};
for node_id in chapter.iter_dfs() {
if let Some(id) = chapter.semantics.id(node_id) {
let key = format!("{}#{}", chapter_path, id);
element_id_map.insert(key, GlobalNodeId::new(*chapter_id, node_id));
}
}
}
if let Ok(mut map) = self.element_id_map.write() {
*map = element_id_map;
}
}
fn resolve_href(&self, from_chapter: ChapterId, href: &str) -> Option<AnchorTarget> {
let from_path = self.source_id(from_chapter)?;
resolve_path_based_href(
from_path,
href,
|p| {
self.chapter_paths
.iter()
.position(|cp| cp == p)
.map(|i| ChapterId(i as u32))
},
|k| {
self.element_id_map
.read()
.ok()
.and_then(|m| m.get(k).copied())
},
)
}
fn resolve_toc(&self) -> Option<Vec<TocEntry>> {
let text = self.cached_text().ok()?;
let (html_start, html_end) = self
.kf8
.flow_table
.first()
.copied()
.unwrap_or((0, text.len()));
let html_text = flow_slice(text, html_start, html_end);
let file_starts: Vec<(u32, u32)> = self
.kf8
.files
.iter()
.map(|f| (f.start_pos, f.file_number as u32))
.collect();
let mut toc = self.toc.clone();
resolve_toc_with_positions(&mut toc, &self.toc_positions, html_text, &file_starts);
Some(toc)
}
}
impl Azw3Importer {
fn cached_text(&self) -> crate::Result<&Vec<u8>> {
if let Some(text) = self.text_cache.get() {
return Ok(text);
}
let _guard = self.text_init.lock().unwrap_or_else(|e| e.into_inner());
if let Some(text) = self.text_cache.get() {
return Ok(text);
}
let text = self.extract_text()?;
Ok(self.text_cache.get_or_init(|| text))
}
}
fn resolve_toc_with_positions(
entries: &mut [TocEntry],
positions: &HashMap<usize, TocPosition>,
html_text: &[u8],
file_starts: &[(u32, u32)],
) {
for entry in entries {
if let Some(pos) = entry.play_order.and_then(|po| positions.get(&po)) {
if let Some(id) = transform::find_nearest_id_fast(
html_text,
pos.byte_pos as usize,
pos.file_num as usize,
file_starts,
) {
if !entry.href.contains('#') {
entry.href = format!("{}#{}", entry.href, id);
}
}
}
resolve_toc_with_positions(&mut entry.children, positions, html_text, file_starts);
}
}
impl Azw3Importer {
pub fn from_source(source: Arc<dyn ByteSource>) -> crate::Result<Self> {
let file_len = source.len();
let header_start = source.read_at(0, 78)?;
if header_start.len() < 78 {
return Err(crate::Error::Malformed {
format: crate::Format::Azw3,
context: "file too short for PDB header".into(),
});
}
let num_records = u16::from_be_bytes([header_start[76], header_start[77]]) as usize;
let header_size = 78 + num_records * 8;
let header_bytes = source.read_at(0, header_size)?;
let (pdb, _) = PdbInfo::parse(&header_bytes)?;
if pdb.num_records < 2 {
return Err(crate::Error::Malformed {
format: crate::Format::Azw3,
context: "not enough PDB records".into(),
});
}
let read_record = |idx: usize| -> io::Result<Vec<u8>> {
let (start, end) = pdb.record_range(idx, file_len)?;
let len = usize::try_from(end - start)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record too large"))?;
source.read_at(start, len)
};
let record0 = read_record(0)?;
let mobi = MobiHeader::parse(&record0)?;
if mobi.encryption != 0 {
return Err(crate::Error::DrmProtected(crate::Format::Azw3));
}
let exth = parse_exth(&record0, &mobi);
let format = detect_format(&mobi, &exth, &pdb, &read_record)?;
let record_offset = format.record_offset();
let mobi = if record_offset > 0 {
let kf8_record0 = read_record(record_offset)?;
MobiHeader::parse(&kf8_record0)?
} else {
mobi
};
if !format.is_kf8() {
return Err(crate::Error::UnsupportedFormat {
detail: "not a KF8/AZW3 file - use MobiImporter for MOBI6 files".into(),
});
}
let metadata = build_metadata(&pdb, &mobi, &exth);
let codec = match mobi.encoding {
Encoding::Utf8 => "utf-8",
_ => "cp1252",
};
let mut read_record_offset = |idx: usize| -> io::Result<Vec<u8>> {
let actual_idx = idx + record_offset;
let (start, end) = pdb.record_range(actual_idx, file_len)?;
let len = usize::try_from(end - start)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record too large"))?;
source.read_at(start, len)
};
let flow_table = if mobi.fdst_index != NULL_INDEX {
let fdst_record = read_record_offset(mobi.fdst_index as usize)?;
parse_fdst(&fdst_record)?
} else {
Vec::new()
};
let files = if mobi.skel_index != NULL_INDEX {
let (entries, _) =
read_index(&mut read_record_offset, mobi.skel_index as usize, codec)?;
parse_skel_index(&entries)
} else {
Vec::new()
};
let elems = if mobi.div_index != NULL_INDEX {
let (entries, cncx) =
read_index(&mut read_record_offset, mobi.div_index as usize, codec)?;
parse_div_index(&entries, &cncx)
} else {
Vec::new()
};
let ncx = if mobi.ncx_index != NULL_INDEX {
let (entries, cncx) =
read_index(&mut read_record_offset, mobi.ncx_index as usize, codec)?;
parse_ncx_index(&entries, &cncx)
} else {
Vec::new()
};
let mut spine = Vec::new();
let mut chapter_paths = Vec::new();
for (i, file) in files.iter().enumerate() {
let filename = format!("part{:04}.html", file.file_number);
chapter_paths.push(filename);
spine.push(SpineEntry {
id: ChapterId(i as u32),
size_estimate: file.length as usize,
});
}
let mut toc_positions = HashMap::new();
let toc = {
let nodes = build_toc_from_ncx(&ncx, |ncx_idx, entry| {
let (file_num, byte_pos) = if let Some((frag_idx, offset)) = entry.pos_fid
&& let Some(elem) = elems.get(frag_idx as usize)
{
(elem.file_number as usize, elem.insert_pos + offset)
} else {
let file_num = find_file_for_position(&files, entry.pos)
.map(|f| f.file_number)
.unwrap_or(0);
(file_num, entry.pos)
};
let chapter_path = format!("part{:04}.html", file_num);
toc_positions.insert(
ncx_idx,
TocPosition {
byte_pos,
file_num: file_num as u32,
},
);
chapter_path
});
nodes.into_iter().map(toc_node_to_entry).collect()
};
let cover_record_idx = exth.and_then(|e| e.cover_offset);
let mut importer = Self {
source,
pdb,
mobi,
record_offset,
file_len,
metadata,
toc,
landmarks: Vec::new(), spine,
chapter_paths,
kf8: Kf8Structure {
flow_table,
files,
elems,
},
text_cache: OnceLock::new(),
parts_cache: OnceLock::new(),
text_init: Mutex::new(()),
chapter_cache: RwLock::new(HashMap::new()),
assets: Vec::new(),
css_cache: RwLock::new(HashMap::new()),
element_id_map: RwLock::new(HashMap::new()),
toc_positions,
};
importer.assets = importer.discover_assets();
if let Some(cover_idx) = cover_record_idx {
let needle = format!("images/image_{cover_idx:04}.");
if let Some(path) = importer.assets.iter().find(|p| p.starts_with(&needle)) {
importer.metadata.cover_image = Some(path.clone());
}
}
Ok(importer)
}
fn extract_text(&self) -> io::Result<Vec<u8>> {
let mut text = Vec::new();
let read_record = |idx: usize| -> io::Result<Vec<u8>> {
let actual_idx = idx + self.record_offset;
let (start, end) = self.pdb.record_range(actual_idx, self.file_len)?;
let len = usize::try_from(end - start)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record too large"))?;
self.source.read_at(start, len)
};
let mut huff_reader = if self.mobi.compression == Compression::Huffman
&& self.mobi.huff_record_index != NULL_INDEX
{
let huff_data = read_record(self.mobi.huff_record_index as usize)?;
let mut cdics = Vec::new();
for i in 0..self.mobi.huff_record_count.saturating_sub(1) {
let cdic_idx = self.mobi.huff_record_index as usize + 1 + i as usize;
if let Ok(cdic) = read_record(cdic_idx) {
cdics.push(cdic);
}
}
let cdic_refs: Vec<&[u8]> = cdics.iter().map(|c| c.as_slice()).collect();
Some(HuffCdicReader::new(&huff_data, &cdic_refs)?)
} else {
None
};
let mut text_budget = crate::mobi::huffcdic::total_text_budget(self.file_len);
for i in 1..=self.mobi.text_record_count as usize {
let record = read_record(i)?;
let stripped = strip_trailing_data(&record, self.mobi.extra_data_flags);
let decompressed = match self.mobi.compression {
Compression::None => stripped.to_vec(),
Compression::PalmDoc => palmdoc::decompress(stripped)?,
Compression::Huffman => {
if let Some(ref mut reader) = huff_reader {
reader.decompress(stripped, &mut text_budget)?
} else {
stripped.to_vec()
}
}
Compression::Unknown(_) => stripped.to_vec(),
};
text.extend_from_slice(&decompressed);
}
Ok(text)
}
fn build_chapter(&self, chapter_id: u32, text: &[u8]) -> io::Result<Vec<u8>> {
let (html_start, html_end) = self
.kf8
.flow_table
.first()
.copied()
.unwrap_or((0, text.len()));
let html_text = flow_slice(text, html_start, html_end);
let parts = self
.parts_cache
.get_or_init(|| build_parts(html_text, &self.kf8.files, &self.kf8.elems));
let content = parts
.get(chapter_id as usize)
.map(|(_, content)| content.clone())
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
format!("Chapter {} not found", chapter_id),
)
})?;
let file_starts: Vec<(u32, u32)> = self
.kf8
.files
.iter()
.map(|f| (f.start_pos, f.file_number as u32))
.collect();
let transformed =
transform::transform_kindle_refs(&content, &self.kf8.elems, html_text, &file_starts);
let mut cleaned = transform::strip_kindle_attributes_fast(&transformed);
crate::mobi::filepos::fix_stray_attribute_solidus(&mut cleaned);
Ok(cleaned)
}
fn discover_assets(&self) -> Vec<String> {
let mut assets = Vec::new();
if self.mobi.first_image_index == NULL_INDEX {
return assets;
}
let first_img = self.mobi.first_image_index as usize + self.record_offset;
for i in first_img..self.pdb.num_records as usize {
if let Ok((start, end)) = self.pdb.record_range(i, self.file_len) {
let read_len = (end - start).min(16) as usize;
let mut header = [0u8; 16];
if self
.source
.read_at_into(start, &mut header[..read_len])
.is_ok()
{
let header = &header[..read_len];
if is_metadata_record(header) {
continue;
}
let idx = i - first_img;
if let Some(media_type) = detect_image_type(header) {
let ext = match media_type {
"image/jpeg" => "jpg",
"image/png" => "png",
"image/gif" => "gif",
_ => "bin",
};
assets.push(format!("images/image_{idx:04}.{ext}"));
} else if let Some(font_ext) = detect_font_type(header) {
assets.push(format!("fonts/font_{idx:04}.{font_ext}"));
}
}
}
}
assets
}
fn load_image_record(&self, idx: usize) -> io::Result<Vec<u8>> {
let first_img = self.mobi.first_image_index as usize + self.record_offset;
let record_idx = first_img + idx;
let data = self.read_record(record_idx)?;
if data.starts_with(b"FONT") {
return decode_font_record(&data);
}
Ok(data)
}
fn read_record(&self, idx: usize) -> io::Result<Vec<u8>> {
let (start, end) = self.pdb.record_range(idx, self.file_len)?;
let len = usize::try_from(end - start)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "record too large"))?;
self.source.read_at(start, len)
}
}
fn detect_format(
mobi: &MobiHeader,
exth: &Option<crate::mobi::ExthHeader>,
pdb: &PdbInfo,
read_record: &dyn Fn(usize) -> io::Result<Vec<u8>>,
) -> io::Result<MobiFormat> {
if mobi.mobi_version == 8 {
return Ok(MobiFormat::Kf8);
}
if let Some(kf8_idx) = exth.as_ref().and_then(|e| e.kf8_boundary) {
let boundary_idx = (kf8_idx as usize).wrapping_sub(1);
if boundary_idx > 0 && boundary_idx < pdb.num_records as usize {
let boundary = read_record(boundary_idx)?;
if boundary.starts_with(b"BOUNDARY") {
return Ok(MobiFormat::Combo {
kf8_record_offset: kf8_idx as usize,
});
}
}
}
Ok(MobiFormat::Mobi6)
}
fn build_metadata(
pdb: &PdbInfo,
mobi: &MobiHeader,
exth: &Option<crate::mobi::ExthHeader>,
) -> Metadata {
let title = exth
.as_ref()
.and_then(|e| e.title.clone())
.or_else(|| {
if !mobi.title.is_empty() {
Some(mobi.title.clone())
} else {
None
}
})
.unwrap_or_else(|| pdb.name.clone());
let mut metadata = Metadata {
title,
..Default::default()
};
if let Some(exth) = exth {
metadata.authors = exth.authors.clone();
metadata.publisher = exth.publisher.clone();
metadata.description = exth.description.clone();
metadata.subjects = exth.subjects.clone();
metadata.date = exth.pub_date.clone();
metadata.rights = exth.rights.clone();
metadata.language = exth.language.clone().unwrap_or_default();
metadata.identifier = exth
.isbn
.clone()
.or_else(|| exth.asin.clone())
.or_else(|| exth.source.clone())
.unwrap_or_default();
}
metadata
}
fn flow_slice(text: &[u8], start: usize, end: usize) -> &[u8] {
let start = start.min(text.len());
let end = end.min(text.len());
if start <= end { &text[start..end] } else { &[] }
}
fn build_parts(
text: &[u8],
files: &[SkeletonFile],
elems: &[DivElement],
) -> Vec<(String, Vec<u8>)> {
let mut parts = Vec::new();
let mut div_ptr = 0;
for file in files {
let skel_start = file.start_pos as usize;
let skel_end = skel_start + file.length as usize;
if skel_end > text.len() {
continue;
}
let mut skeleton = text[skel_start..skel_end].to_vec();
let mut baseptr = skel_end;
for _i in 0..file.div_count {
if div_ptr >= elems.len() {
break;
}
let elem = &elems[div_ptr];
let part_len = elem.length as usize;
if baseptr + part_len > text.len() {
div_ptr += 1;
continue;
}
let part = &text[baseptr..baseptr + part_len];
let insert_pos = (elem.insert_pos as usize).saturating_sub(skel_start);
if insert_pos <= skeleton.len() {
let mut new_skeleton = Vec::with_capacity(skeleton.len() + part.len());
new_skeleton.extend_from_slice(&skeleton[..insert_pos]);
new_skeleton.extend_from_slice(part);
new_skeleton.extend_from_slice(&skeleton[insert_pos..]);
skeleton = new_skeleton;
}
baseptr += part_len;
div_ptr += 1;
}
let filename = format!("part{:04}.html", file.file_number);
parts.push((filename, skeleton));
}
if parts.is_empty() && !text.is_empty() {
parts.push(("part0000.html".to_string(), text.to_vec()));
}
parts
}
fn find_file_for_position(files: &[SkeletonFile], pos: u32) -> Option<&SkeletonFile> {
for file in files {
if pos >= file.start_pos && pos < file.start_pos + file.length {
return Some(file);
}
}
files.first()
}
fn toc_node_to_entry(node: TocNode) -> TocEntry {
let mut entry = TocEntry::new(&node.title, &node.href);
entry.play_order = Some(node.ncx_index);
entry.children = node.children.into_iter().map(toc_node_to_entry).collect();
entry
}