boko 0.5.0

Fast native ebook converter for EPUB, KFX, AZW3, and MOBI — the only KFX writer that needs no Kindle Previewer
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
use super::*;

/// Build chapter entities returning them separately for grouped emission.
///
/// Returns (section, storyline). Text content accumulates in the book-global
/// content chunks (`ExportContext::append_text`); the resulting `$145`
/// fragments are emitted once after all chapters via
/// [`build_content_fragments`].
pub(super) fn build_chapter_entities_grouped(
    chapter: &Chapter,
    chapter_id: ChapterId,
    section_name: &str,
    ctx: &mut ExportContext,
) -> (KfxFragment, KfxFragment) {
    use crate::kfx::storyline::{ir_to_tokens, tokens_to_ion};

    // Check if this is a cover chapter (image-only)
    // Only treat as cover if there's no standalone cover section (c0)
    // When ctx.cover_fragment_id is set, c0 already handles the cover
    let is_cover = ctx.cover_fragment_id.is_none() && is_image_only_chapter(chapter);

    // =========================================================================
    // 1. SETUP: Naming for this chapter's entity pair
    // =========================================================================
    let story_name = format!("story_{}", section_name);

    let section_name_symbol = ctx.symbols.get_or_intern(section_name);
    let story_name_symbol = ctx.symbols.get_or_intern(&story_name);

    ctx.begin_chapter();

    // Get the section fragment ID assigned during Pass 1
    let section_id = ctx
        .get_chapter_fragment(chapter_id)
        .unwrap_or_else(|| ctx.next_fragment_id());

    // =========================================================================
    // 2. GENERATE: Schema-driven token generation
    // =========================================================================
    let storyline_content_list = if is_cover {
        // For cover chapters, generate flat storyline with direct image
        build_cover_storyline(chapter, ctx)
    } else {
        // Normal chapter: full token-based generation
        let tokens = ir_to_tokens(chapter, ctx);
        tokens_to_ion(&tokens, ctx)
    };

    // =========================================================================
    // 3. ASSEMBLE: Package into two KFX Entities
    // =========================================================================

    // STORYLINE ($259) - Holds the structure, references Content by name
    let storyline_ion = IonValue::Struct(vec![
        (
            KfxSymbol::StoryName as u64,
            IonValue::Symbol(story_name_symbol),
        ),
        (KfxSymbol::ContentList as u64, storyline_content_list),
    ]);
    let storyline_fragment = KfxFragment::new(KfxSymbol::Storyline, &story_name, storyline_ion);

    // Entity C: SECTION ($260) - Entry point, references Storyline by story_name
    let page_template = if is_cover {
        // Cover page: container type with fixed dimensions and scale_fit layout
        IonValue::Struct(vec![
            (KfxSymbol::Id as u64, IonValue::Int(section_id as i64)),
            (
                KfxSymbol::StoryName as u64,
                IonValue::Symbol(story_name_symbol),
            ),
            (
                KfxSymbol::Type as u64,
                IonValue::Symbol(KfxSymbol::Container as u64),
            ),
            (KfxSymbol::FixedWidth as u64, IonValue::Int(1400)),
            (KfxSymbol::FixedHeight as u64, IonValue::Int(2100)),
            (
                KfxSymbol::Layout as u64,
                IonValue::Symbol(KfxSymbol::ScaleFit as u64),
            ),
            (
                KfxSymbol::Float as u64,
                IonValue::Symbol(KfxSymbol::Center as u64),
            ),
        ])
    } else {
        // Normal text page
        IonValue::Struct(vec![
            (KfxSymbol::Id as u64, IonValue::Int(section_id as i64)),
            (
                KfxSymbol::StoryName as u64,
                IonValue::Symbol(story_name_symbol),
            ),
            (
                KfxSymbol::Type as u64,
                IonValue::Symbol(KfxSymbol::Text as u64),
            ),
        ])
    };

    let section_ion = IonValue::Struct(vec![
        (
            KfxSymbol::SectionName as u64,
            IonValue::Symbol(section_name_symbol),
        ),
        (
            KfxSymbol::PageTemplates as u64,
            IonValue::List(vec![page_template]),
        ),
    ]);
    let section_fragment = KfxFragment::new(KfxSymbol::Section, section_name, section_ion);

    (section_fragment, storyline_fragment)
}

/// Build the book-global `$145` content fragments from the accumulated chunks.
///
/// Called once after every chapter's storyline has been generated; each chunk
/// is at most [`crate::kfx::context::MAX_CONTENT_CHUNK_BYTES`] of UTF-8 text.
pub(super) fn build_content_fragments(ctx: &mut ExportContext) -> Vec<KfxFragment> {
    ctx.take_content_chunks()
        .into_iter()
        .map(|(name, segments)| {
            let name_symbol = ctx.symbols.get_or_intern(&name);
            let content_ion = IonValue::Struct(vec![
                (KfxSymbol::Name as u64, IonValue::Symbol(name_symbol)),
                (
                    KfxSymbol::ContentList as u64,
                    IonValue::List(segments.into_iter().map(IonValue::String).collect()),
                ),
            ]);
            KfxFragment::new(KfxSymbol::Content, &name, content_ion)
        })
        .collect()
}

/// Build a simplified storyline for cover chapters.
///
/// Cover pages have a flat structure with just the image directly in content_list,
/// no container wrapper. Structure: [{ type: image, resource_name, style }]
pub(super) fn build_cover_storyline(chapter: &Chapter, ctx: &mut ExportContext) -> IonValue {
    use crate::model::Role;

    // Find the image node
    for node_id in chapter.iter_dfs() {
        let node = match chapter.node(node_id) {
            Some(n) => n,
            None => continue,
        };

        if node.role == Role::Image {
            // Get the image source
            if let Some(src) = chapter.semantics.src(node_id) {
                // Look up the resource name (e.g., "e0")
                let resource_name = ctx.resource_registry.get_or_create_name(src);
                let resource_name_symbol = ctx.symbols.get_or_intern(&resource_name);

                // Use the default (empty) style, not the node's CSS style:
                // scale-fit page templates carry all positioning themselves
                // (fixed dims + scale_fit + float center), and readers require
                // the image element to add no properties of its own — any
                // leftover (max-width, borders, font shorthand) is flagged as
                // an unexpected image style. Kindle Previewer does the same:
                // its scale-fit images reference an empty style.
                let style_symbol = ctx.default_style_symbol;
                ctx.default_style_used = true;

                // Generate unique container ID
                let container_id = ctx.fragment_ids.next_id();

                // Record content ID for position_map and location_map
                ctx.record_content_id(container_id);
                // Record length of 1 for image (per kfx_output algorithm)
                ctx.record_content_length(container_id, 1);

                // Build the image struct directly (no container wrapper)
                let image_struct = IonValue::Struct(vec![
                    (KfxSymbol::Id as u64, IonValue::Int(container_id as i64)),
                    (KfxSymbol::Style as u64, IonValue::Symbol(style_symbol)),
                    (
                        KfxSymbol::Type as u64,
                        IonValue::Symbol(KfxSymbol::Image as u64),
                    ),
                    (
                        KfxSymbol::ResourceName as u64,
                        IonValue::Symbol(resource_name_symbol),
                    ),
                ]);

                return IonValue::List(vec![image_struct]);
            }
        }
    }

    // Fallback: empty list if no image found
    IonValue::List(vec![])
}

/// Build the document symbols section.
///
/// This writes the local symbol table in the format expected by KFX readers:
/// ```ion
/// $ion_symbol_table::{
///   imports: [{ name: "YJ_symbols", version: 10, max_id: 851 }],
///   symbols: ["local_sym1", "local_sym2", ...]
/// }
/// ```
///
/// Ion system symbol IDs:
/// - $3 = $ion_symbol_table
/// - $4 = name
/// - $5 = version
/// - $6 = imports
/// - $7 = symbols
/// - $8 = max_id
///
/// IMPORTANT: The symbols in the list must appear in the exact same order
/// they were interned, so that symbol ID = KFX_SYMBOL_TABLE_SIZE + index.
pub(super) fn build_symbol_table_ion(local_symbols: &[String]) -> Vec<u8> {
    use crate::kfx::ion::IonWriter;
    use crate::kfx::symbols::KFX_MAX_SYMBOL_ID;

    let mut writer = IonWriter::new();
    writer.write_bvm();

    // Build the import entry for YJ_symbols (Amazon's KFX symbol table)
    // { name: "YJ_symbols", version: 10, max_id: 851 }
    let import_entry = IonValue::Struct(vec![
        (4, IonValue::String("YJ_symbols".to_string())), // $4 = name
        (5, IonValue::Int(10)),                          // $5 = version
        (8, IonValue::Int(KFX_MAX_SYMBOL_ID as i64)),    // $8 = max_id
    ]);

    // Build the symbols list with local symbols
    let symbols_list: Vec<IonValue> = local_symbols
        .iter()
        .map(|s| IonValue::String(s.clone()))
        .collect();

    // Build the $ion_symbol_table struct
    // { imports: [...], symbols: [...] }
    let symbol_table = IonValue::Struct(vec![
        (6, IonValue::List(vec![import_entry])), // $6 = imports
        (7, IonValue::List(symbols_list)),       // $7 = symbols
    ]);

    // Write with $ion_symbol_table annotation ($3)
    writer.write_annotated(&[3], &symbol_table);

    writer.into_bytes()
}

/// Build format capabilities ION.
pub(super) fn build_format_capabilities_ion(has_text_content: bool) -> Vec<u8> {
    // Reference shape (Kindle Previewer output, tests/fixtures/epictetus.kfx):
    // a LIST of `{$492: <capability key>, version: <int>}` structs. Readers
    // walk it with per-entry key lookups, so any other shape breaks them.
    // `kfxgen.textBlock 1` declares that text lives in $145 content
    // fragments; an image-only book (no $145 at all) must not declare it —
    // capability/content mismatches are conformance errors. The flat
    // $264/$265 position maps with no eid offsets declare neither
    // `kfxgen.positionMaps` nor `pidMapWithOffset`, matching the reference.
    let mut caps = Vec::new();
    if has_text_content {
        caps.push(IonValue::Struct(vec![
            (
                KfxSymbol::Key as u64,
                IonValue::String("kfxgen.textBlock".to_string()),
            ),
            (KfxSymbol::Version as u64, IonValue::Int(1)),
        ]));
    }
    let caps = IonValue::List(caps);

    // Annotate with $593 (format_capabilities)
    serialize_annotated_ion(KfxSymbol::FormatCapabilities as u64, &caps)
}

#[cfg(test)]
#[allow(clippy::vec_init_then_push, clippy::needless_range_loop)]
mod entity_structure_tests {
    use super::*;
    use crate::kfx::fragment::FragmentData;
    use crate::model::Book;

    #[test]
    fn test_entity_order_matches_reference() {
        // Build KFX from EPUB and verify entity order matches Amazon reference
        let book = Book::open("tests/fixtures/epictetus.epub").unwrap();
        let container_id = generate_container_id("test-seed");
        let mut ctx = ExportContext::new();

        // Collect spine info
        let spine_info: Vec<_> = book
            .spine()
            .iter()
            .enumerate()
            .map(|(idx, entry)| {
                let section_name = format!("c{}", idx);
                (entry.id, section_name)
            })
            .collect();

        // Pass 1: Survey
        for (chapter_id, section_name) in &spine_info {
            ctx.register_section(section_name);
            let source_path = book.source_id(*chapter_id).unwrap_or("").to_string();
            if let Ok(chapter) = book.load_chapter(*chapter_id) {
                survey_chapter(&chapter, *chapter_id, &source_path, &mut ctx);
            }
        }

        // Pass 2: Build fragments in correct order
        let mut fragments = Vec::new();

        fragments.push(build_content_features_fragment(&ctx));
        fragments.push(build_book_metadata_fragment(&book, &container_id, &ctx));
        fragments.push(build_metadata_fragment(&ctx));
        fragments.push(build_document_data_fragment(&ctx));
        fragments.push(build_book_navigation_fragment_with_positions(&book, &ctx));

        let mut section_fragments = Vec::new();
        let mut storyline_fragments = Vec::new();
        let mut content_fragments = Vec::new();

        for (chapter_id, section_name) in &spine_info {
            if let Ok(chapter) = book.load_chapter(*chapter_id) {
                let (section, storyline) =
                    build_chapter_entities_grouped(&chapter, *chapter_id, section_name, &mut ctx);
                section_fragments.push(section);
                storyline_fragments.push(storyline);
            }
        }
        content_fragments.extend(build_content_fragments(&mut ctx));

        fragments.extend(section_fragments);
        fragments.extend(storyline_fragments);
        fragments.extend(content_fragments);

        // Verify entity type order matches reference pattern:
        // content_features, book_metadata, metadata, document_data, book_navigation,
        // sections (grouped), storylines (grouped), content (grouped)

        let types: Vec<u64> = fragments.iter().map(|f| f.ftype).collect();

        // First 5 should be the header entities in order
        assert_eq!(types[0], KfxSymbol::ContentFeatures as u64);
        assert_eq!(types[1], KfxSymbol::BookMetadata as u64);
        assert_eq!(types[2], KfxSymbol::Metadata as u64);
        assert_eq!(types[3], KfxSymbol::DocumentData as u64);
        assert_eq!(types[4], KfxSymbol::BookNavigation as u64);

        // After header, all sections should come first, then storylines, then content
        let after_header = &types[5..];
        let section_count = after_header
            .iter()
            .take_while(|&&t| t == KfxSymbol::Section as u64)
            .count();
        assert!(section_count > 0, "should have sections after header");

        let after_sections = &after_header[section_count..];
        let storyline_count = after_sections
            .iter()
            .take_while(|&&t| t == KfxSymbol::Storyline as u64)
            .count();
        assert!(storyline_count > 0, "should have storylines after sections");

        let after_storylines = &after_sections[storyline_count..];
        let content_count = after_storylines
            .iter()
            .take_while(|&&t| t == KfxSymbol::Content as u64)
            .count();
        // Content is optional (image-only chapters may not have content)
        // Just verify that after storylines, we only have content entities (if any)
        for t in after_storylines.iter().take(content_count) {
            assert_eq!(
                *t,
                KfxSymbol::Content as u64,
                "content should follow storylines"
            );
        }

        // Verify grouping - no interleaving
        for i in 1..section_count {
            assert_eq!(
                after_header[i],
                KfxSymbol::Section as u64,
                "sections should be grouped"
            );
        }
        for i in 1..storyline_count {
            assert_eq!(
                after_sections[i],
                KfxSymbol::Storyline as u64,
                "storylines should be grouped"
            );
        }
    }

    #[test]
    fn test_chapter_entities_grouped_returns_correct_types() {
        let book = Book::open("tests/fixtures/epictetus.epub").unwrap();
        let mut ctx = ExportContext::new();

        // Get first chapter
        let spine_entry = book.spine().first().unwrap();
        let chapter_id = spine_entry.id;
        let section_name = "c0";
        ctx.register_section(section_name);

        // Survey chapter first
        let source_path = book.source_id(chapter_id).unwrap_or("").to_string();
        if let Ok(chapter) = book.load_chapter(chapter_id) {
            survey_chapter(&chapter, chapter_id, &source_path, &mut ctx);
        }

        // Build entities
        let chapter = book.load_chapter(chapter_id).unwrap();
        let (section, storyline) =
            build_chapter_entities_grouped(&chapter, chapter_id, section_name, &mut ctx);
        let content_fragments = build_content_fragments(&mut ctx);

        // Verify types
        assert_eq!(section.ftype, KfxSymbol::Section as u64);
        assert_eq!(storyline.ftype, KfxSymbol::Storyline as u64);

        // Verify section has section_name and page_templates
        if let FragmentData::Ion(IonValue::Struct(fields)) = &section.data {
            let has_section_name = fields
                .iter()
                .any(|(id, _)| *id == KfxSymbol::SectionName as u64);
            let has_page_templates = fields
                .iter()
                .any(|(id, _)| *id == KfxSymbol::PageTemplates as u64);
            assert!(has_section_name, "section should have section_name");
            assert!(has_page_templates, "section should have page_templates");
        }

        // Verify storyline has story_name and content_list
        if let FragmentData::Ion(IonValue::Struct(fields)) = &storyline.data {
            let has_story_name = fields
                .iter()
                .any(|(id, _)| *id == KfxSymbol::StoryName as u64);
            let has_content_list = fields
                .iter()
                .any(|(id, _)| *id == KfxSymbol::ContentList as u64);
            assert!(has_story_name, "storyline should have story_name");
            assert!(has_content_list, "storyline should have content_list");
        }

        // Content chunks are optional (image-only chapters produce none), but
        // any that exist must carry a name and content_list.
        for content_frag in &content_fragments {
            assert_eq!(content_frag.ftype, KfxSymbol::Content as u64);
            if let FragmentData::Ion(IonValue::Struct(fields)) = &content_frag.data {
                let has_name = fields.iter().any(|(id, _)| *id == KfxSymbol::Name as u64);
                let has_content_list = fields
                    .iter()
                    .any(|(id, _)| *id == KfxSymbol::ContentList as u64);
                assert!(has_name, "content should have name");
                assert!(has_content_list, "content should have content_list");
            }
        }
    }
}

#[cfg(test)]
mod section_type_tests {
    use super::*;
    use crate::kfx::cover::{needs_standalone_cover, normalize_cover_path};
    use crate::kfx::fragment::FragmentData;
    use crate::model::Book;

    /// When a standalone cover (c0) exists, the titlepage chapter (c1) should have
    /// type: text, NOT type: container. The container type is reserved for c0.
    #[test]
    fn test_titlepage_section_has_text_type_when_standalone_cover_exists() {
        let book = Book::open("tests/fixtures/epictetus.epub").unwrap();
        let mut ctx = ExportContext::new();

        // Verify this book needs a standalone cover (cover.jpg != titlepage.png)
        let asset_paths = book.list_assets();
        let cover_image = book
            .metadata()
            .cover_image
            .clone()
            .expect("should have cover");
        let normalized = normalize_cover_path(&cover_image, asset_paths);

        // Get first chapter ID
        let first_chapter_id = book.spine().first().expect("should have spine").id;
        let first_chapter = book.load_chapter(first_chapter_id).unwrap();
        assert!(
            needs_standalone_cover(&normalized, &first_chapter),
            "test requires a book with different cover and titlepage images"
        );

        // Register c0 for standalone cover, c1 for titlepage
        ctx.register_section("c0");
        ctx.register_section("c1");
        ctx.cover_fragment_id = Some(ctx.next_fragment_id()); // Mark that standalone cover exists

        // Survey the titlepage chapter
        let source_path = book.source_id(first_chapter_id).unwrap_or("").to_string();
        let first_chapter = book.load_chapter(first_chapter_id).unwrap();
        survey_chapter(&first_chapter, first_chapter_id, &source_path, &mut ctx);

        // Build the titlepage section (c1)
        let first_chapter = book.load_chapter(first_chapter_id).unwrap();
        let (section, _) =
            build_chapter_entities_grouped(&first_chapter, first_chapter_id, "c1", &mut ctx);

        // Extract the page_template type from the section
        if let FragmentData::Ion(IonValue::Struct(fields)) = &section.data {
            let page_templates = fields
                .iter()
                .find(|(id, _)| *id == KfxSymbol::PageTemplates as u64)
                .expect("section should have page_templates");

            if let (_, IonValue::List(templates)) = page_templates {
                let template = &templates[0];
                if let IonValue::Struct(template_fields) = template {
                    let type_field = template_fields
                        .iter()
                        .find(|(id, _)| *id == KfxSymbol::Type as u64)
                        .expect("page_template should have type");

                    if let (_, IonValue::Symbol(type_sym)) = type_field {
                        assert_eq!(
                            *type_sym,
                            KfxSymbol::Text as u64,
                            "titlepage (c1) should have type: text when standalone cover exists, \
                             but got type: container"
                        );
                    } else {
                        panic!("type should be a symbol");
                    }
                }
            }
        } else {
            panic!("section should have Ion struct data");
        }
    }
}