text-document-io 1.9.1

Import/export for text-document: plain text, Markdown, HTML, LaTeX, DOCX
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
// Generated by Qleany v1.4.8 from feature_use_case.tera
use crate::ImportHtmlDto;
use crate::ImportHtmlResultDto;
use anyhow::{Result, anyhow};
use common::database::CommandUnitOfWork;
use common::database::rope_helpers::{
    rope_append_block, rope_append_table_anchor, rope_insert_block_boundary, rope_reset,
};
use common::entities::{
    Block, Document, Frame, FramePosition, List, Resource, Root, Table, TableCell,
};

use common::long_operation::LongOperation;
use common::parser_tools::content_parser::{
    ParsedElement, ParsedInline, format_runs_from_spans, parse_html_elements,
};
use common::parser_tools::list_grouper::ListGrouper;
use common::types::{EntityId, ROOT_ENTITY_ID};
use std::sync::Arc;

pub trait ImportHtmlUnitOfWorkFactoryTrait: Send + Sync {
    fn create(&self) -> Box<dyn ImportHtmlUnitOfWorkTrait>;
}

#[macros::uow_action(entity = "Root", action = "Get", thread_safe = true)]
#[macros::uow_action(entity = "Root", action = "GetRelationship", thread_safe = true)]
#[macros::uow_action(entity = "Document", action = "Get", thread_safe = true)]
#[macros::uow_action(entity = "Document", action = "Update", thread_safe = true)]
#[macros::uow_action(entity = "Document", action = "GetRelationship", thread_safe = true)]
#[macros::uow_action(entity = "Frame", action = "Get", thread_safe = true)]
#[macros::uow_action(entity = "Frame", action = "Create", thread_safe = true)]
#[macros::uow_action(entity = "Frame", action = "Update", thread_safe = true)]
#[macros::uow_action(
    entity = "Frame",
    action = "UpdateWithRelationships",
    thread_safe = true
)]
#[macros::uow_action(entity = "Frame", action = "Remove", thread_safe = true)]
#[macros::uow_action(entity = "Frame", action = "GetRelationship", thread_safe = true)]
#[macros::uow_action(entity = "Block", action = "Create", thread_safe = true)]
#[macros::uow_action(entity = "Block", action = "SetRelationship", thread_safe = true)]
#[macros::uow_action(entity = "List", action = "Create", thread_safe = true)]
#[macros::uow_action(entity = "Resource", action = "Create", thread_safe = true)]
#[macros::uow_action(entity = "Table", action = "Create", thread_safe = true)]
#[macros::uow_action(entity = "TableCell", action = "Create", thread_safe = true)]
pub trait ImportHtmlUnitOfWorkTrait: CommandUnitOfWork + Send + Sync {}

pub struct ImportHtmlUseCase {
    uow_factory: Box<dyn ImportHtmlUnitOfWorkFactoryTrait>,
    dto: ImportHtmlDto,
}

struct FrameState {
    frame_id: EntityId,
    child_order: Vec<i64>,
}

/// Advance the blockquote frame stack to match `target_depth`, finalising
/// frames that close (writing back their `child_order`) and creating new
/// blockquote sub-frames as depth increases. Resets `list_grouper` on every
/// frame boundary so lists never group across blockquote boundaries.
///
/// Touches only Frame entities — never the rope — so rope mirroring and
/// `document_position` bookkeeping in the caller are unaffected.
fn transition_bq_depth(
    uow: &mut Box<dyn ImportHtmlUnitOfWorkTrait>,
    doc_id: EntityId,
    frame_stack: &mut Vec<FrameState>,
    current_bq_depth: &mut u32,
    target_depth: u32,
    list_grouper: &mut ListGrouper,
) -> Result<()> {
    // Close blockquote frames if depth decreased
    while *current_bq_depth > target_depth && frame_stack.len() > 1 {
        let finished = frame_stack.pop().unwrap();
        let mut frame_entity = uow
            .get_frame(&finished.frame_id)?
            .ok_or_else(|| anyhow!("Blockquote frame not found"))?;
        frame_entity.child_order = finished.child_order;
        uow.update_frame(&frame_entity)?;
        *current_bq_depth -= 1;
        list_grouper.reset();
    }

    // Open blockquote frames if depth increased
    while *current_bq_depth < target_depth {
        let parent_frame_id = frame_stack.last().unwrap().frame_id;
        let bq_frame = Frame {
            fmt_is_blockquote: Some(true),
            fmt_position: Some(FramePosition::InFlow),
            parent_frame: Some(parent_frame_id),
            ..Frame::default()
        };
        let created_bq = uow.create_frame(&bq_frame, doc_id, -1)?;
        frame_stack
            .last_mut()
            .unwrap()
            .child_order
            .push(-(created_bq.id as i64));
        frame_stack.push(FrameState {
            frame_id: created_bq.id,
            child_order: Vec::new(),
        });
        *current_bq_depth += 1;
        list_grouper.reset();
    }

    Ok(())
}

impl ImportHtmlUseCase {
    pub fn new(
        uow_factory: Box<dyn ImportHtmlUnitOfWorkFactoryTrait>,
        dto: &ImportHtmlDto,
    ) -> Self {
        ImportHtmlUseCase {
            uow_factory,
            dto: dto.clone(),
        }
    }
}

impl LongOperation for ImportHtmlUseCase {
    type Output = ImportHtmlResultDto;

    fn execute(
        &self,
        progress_callback: Box<dyn Fn(common::long_operation::OperationProgress) + Send>,
        cancel_flag: Arc<std::sync::atomic::AtomicBool>,
    ) -> Result<Self::Output> {
        use std::sync::atomic::Ordering;

        progress_callback(common::long_operation::OperationProgress::new(
            0.0,
            Some("Starting HTML import...".to_string()),
        ));

        // Parse HTML
        let parsed_elements = parse_html_elements(&self.dto.html_text);

        progress_callback(common::long_operation::OperationProgress::new(
            10.0,
            Some("Parsed HTML, building document...".to_string()),
        ));

        let mut uow = self.uow_factory.create();
        uow.begin_transaction()?;

        // Step 1: Get Root and Document
        let root = uow
            .get_root(&ROOT_ENTITY_ID)?
            .ok_or_else(|| anyhow!("Root entity not found"))?;

        let doc_ids = uow.get_root_relationship(
            &root.id,
            &common::direct_access::root::RootRelationshipField::Document,
        )?;
        let doc_id = *doc_ids
            .first()
            .ok_or_else(|| anyhow!("Root has no associated Document"))?;

        // Step 2: Remove existing frames
        let frame_ids = uow.get_document_relationship(
            &doc_id,
            &common::direct_access::document::DocumentRelationshipField::Frames,
        )?;
        for frame_id in &frame_ids {
            uow.remove_frame(frame_id)?;
        }

        if cancel_flag.load(Ordering::Relaxed) {
            uow.rollback()?;
            return Err(anyhow!("Operation was cancelled"));
        }

        progress_callback(common::long_operation::OperationProgress::new(
            20.0,
            Some("Creating new document structure...".to_string()),
        ));

        // Step 3: Create root frame
        let new_frame = Frame::default();
        let created_frame = uow.create_frame(&new_frame, doc_id, -1)?;

        // Importers replace the entire document — reset the rope+
        // block_offsets. No-op under default backend.
        rope_reset(&uow.store());

        // Step 4: Create blocks with format runs and image anchors
        // Track blockquote frame stack
        let total_elements = parsed_elements.len();
        let mut total_chars: i64 = 0;
        let mut total_block_count: i64 = 0;
        let mut document_position: i64 = 0;

        let mut frame_stack: Vec<FrameState> = vec![FrameState {
            frame_id: created_frame.id,
            child_order: Vec::new(),
        }];
        let mut current_bq_depth: u32 = 0;
        let mut list_grouper = ListGrouper::new();
        // Same intent as in import_markdown_uc: rope inter-block
        // boundaries go between main-flow blocks, not before the
        // first and not for table-cell blocks (deferred to step 5.5).
        let mut emitted_any_main_block = false;

        for (i, parsed_element) in parsed_elements.iter().enumerate() {
            if cancel_flag.load(Ordering::Relaxed) {
                uow.rollback()?;
                return Err(anyhow!("Operation was cancelled"));
            }

            match parsed_element {
                // `parse_html_elements` has no footnote syntax to recognise (HTML has
                // none of its own — `[^label]`/`[^label]:` are a Markdown/Djot
                // convention), so it never emits this variant. Dropping it here is
                // therefore unreachable today, and stated rather than assumed. Markdown
                // and Djot both build a detached `footnote_label` frame instead — see
                // `import_markdown_uc`/`import_djot_uc`.
                ParsedElement::FootnoteDefinition { .. } => {}
                ParsedElement::Block(parsed_block) => {
                    transition_bq_depth(
                        &mut uow,
                        doc_id,
                        &mut frame_stack,
                        &mut current_bq_depth,
                        parsed_block.blockquote_depth,
                        &mut list_grouper,
                    )?;

                    let ParsedInline {
                        plain_text,
                        runs: format_runs,
                        images: block_images,
                        footnote_refs: block_footnote_refs,
                    } = format_runs_from_spans(&parsed_block.spans, parsed_block.is_code_block);
                    let line_len = plain_text.chars().count() as i64;

                    let current_frame_id = frame_stack.last().unwrap().frame_id;
                    let block = Block {
                        document_position,
                        fmt_heading_level: parsed_block.heading_level,
                        fmt_line_height: parsed_block.line_height,
                        fmt_non_breakable_lines: parsed_block.non_breakable_lines,
                        fmt_direction: parsed_block.direction.clone(),
                        fmt_background_color: parsed_block.background_color.clone(),
                        fmt_is_code_block: if parsed_block.is_code_block {
                            Some(true)
                        } else {
                            None
                        },
                        fmt_code_language: parsed_block.code_language.clone(),
                        ..Block::default()
                    };

                    let created_block = uow.create_block(&block, current_frame_id, -1)?;

                    // Mirror into the global rope. Inter-block `\n`
                    // before every block after the first.
                    if emitted_any_main_block {
                        rope_insert_block_boundary(&uow.store());
                    }
                    rope_append_block(&uow.store(), created_block.id, &plain_text);
                    emitted_any_main_block = true;

                    // Write format_runs directly; block_images stays empty for
                    // HTML import (parser does not surface inline images today).
                    {
                        let store = uow.store();
                        let mut runs_map = store.format_runs.write();
                        if !format_runs.is_empty() {
                            runs_map.insert(created_block.id, format_runs);
                        } else {
                            runs_map.remove(&created_block.id);
                        }
                        // Image anchors travel with the runs: both index into the same
                        // block text, which already carries each image's U+FFFC sentinel.
                        let mut images_map = store.block_images.write();
                        if !block_images.is_empty() {
                            images_map.insert(created_block.id, block_images);
                        } else {
                            images_map.remove(&created_block.id);
                        }
                        // Footnote references index into the same block text.
                        let mut notes_map = store.block_footnote_refs.write();
                        if !block_footnote_refs.is_empty() {
                            notes_map.insert(created_block.id, block_footnote_refs);
                        } else {
                            notes_map.remove(&created_block.id);
                        }
                    }

                    // Handle list items
                    if let Some(ref list_style) = parsed_block.list_style {
                        let list_id = if let Some(existing_id) =
                            list_grouper.try_reuse(list_style, parsed_block.list_indent)
                        {
                            existing_id
                        } else {
                            let list = List {
                                style: list_style.clone(),
                                indent: parsed_block.list_indent as i64,
                                ..List::default()
                            };
                            let created_list = uow.create_list(&list, doc_id, -1)?;
                            list_grouper.register(
                                created_list.id,
                                list_style.clone(),
                                parsed_block.list_indent,
                            );
                            created_list.id
                        };

                        uow.set_block_relationship(
                            &created_block.id,
                            &common::direct_access::block::BlockRelationshipField::List,
                            &[list_id],
                        )?;
                    } else {
                        list_grouper.reset();
                    }

                    frame_stack
                        .last_mut()
                        .unwrap()
                        .child_order
                        .push(created_block.id as i64);
                    total_chars += line_len;
                    total_block_count += 1;

                    document_position += line_len;
                    if i < total_elements - 1 {
                        document_position += 1;
                    }
                }

                ParsedElement::Table(parsed_table) => {
                    // Synchronise the blockquote frame stack with the table's
                    // depth so a table inside (or right after) a blockquote
                    // lands in the correct frame.
                    transition_bq_depth(
                        &mut uow,
                        doc_id,
                        &mut frame_stack,
                        &mut current_bq_depth,
                        parsed_table.blockquote_depth,
                        &mut list_grouper,
                    )?;

                    // A table always interrupts a list, regardless of depth.
                    list_grouper.reset();
                    let num_rows = parsed_table.rows.len() as i64;
                    let num_cols = parsed_table.rows.first().map_or(0, |r| r.len()) as i64;
                    if num_rows == 0 || num_cols == 0 {
                        continue;
                    }

                    // 1. Create Table entity (owned by Document)
                    let table = Table {
                        rows: num_rows,
                        columns: num_cols,
                        column_widths: vec![],
                        ..Table::default()
                    };
                    let created_table = uow.create_table(&table, doc_id, -1)?;

                    // 1b. Mirror the table-anchor sentinel into the
                    // global rope. Appended at the end (the importer
                    // processes elements linearly). The boundary comes
                    // from the same positional flag every block uses —
                    // see the note on `rope_append_table_anchor`.
                    rope_append_table_anchor(
                        &uow.store(),
                        created_table.id,
                        emitted_any_main_block,
                    );

                    // 2. Create cell frames with content + TableCell entities
                    let current_frame_id = frame_stack.last().unwrap().frame_id;
                    let total_cells = num_rows * num_cols;
                    let mut cell_count: i64 = 0;
                    let mut created_cell_frame_ids: Vec<EntityId> = Vec::new();

                    for (r, row) in parsed_table.rows.iter().enumerate() {
                        for (c, cell) in row.iter().enumerate() {
                            let cell_frame = Frame::default();
                            let created_cell_frame = uow.create_frame(&cell_frame, doc_id, -1)?;
                            created_cell_frame_ids.push(created_cell_frame.id);

                            let ParsedInline {
                                plain_text,
                                runs: format_runs,
                                images: block_images,
                                footnote_refs: block_footnote_refs,
                            } = format_runs_from_spans(&cell.spans, false);

                            let block = Block {
                                document_position,
                                ..Block::default()
                            };
                            let created_block =
                                uow.create_block(&block, created_cell_frame.id, -1)?;

                            {
                                let store = uow.store();
                                let mut runs_map = store.format_runs.write();
                                if !format_runs.is_empty() {
                                    runs_map.insert(created_block.id, format_runs);
                                } else {
                                    runs_map.remove(&created_block.id);
                                }
                                // Image anchors travel with the runs: both index into the same
                                // block text, which already carries each image's U+FFFC sentinel.
                                let mut images_map = store.block_images.write();
                                if !block_images.is_empty() {
                                    images_map.insert(created_block.id, block_images);
                                } else {
                                    images_map.remove(&created_block.id);
                                }
                                // Footnote references index into the same block text.
                                let mut notes_map = store.block_footnote_refs.write();
                                if !block_footnote_refs.is_empty() {
                                    notes_map.insert(created_block.id, block_footnote_refs);
                                } else {
                                    notes_map.remove(&created_block.id);
                                }
                            }

                            // Mirror the cell block into the global
                            // rope: insert a `\n` boundary, then the
                            // cell's content.
                            rope_insert_block_boundary(&uow.store());
                            rope_append_block(&uow.store(), created_block.id, &plain_text);

                            let mut updated_cell_frame = created_cell_frame.clone();
                            updated_cell_frame.child_order = vec![created_block.id as i64];
                            uow.update_frame(&updated_cell_frame)?;

                            let table_cell = TableCell {
                                row: r as i64,
                                column: c as i64,
                                row_span: 1,
                                column_span: 1,
                                cell_frame: Some(created_cell_frame.id),
                                ..TableCell::default()
                            };
                            uow.create_table_cell(&table_cell, created_table.id, -1)?;

                            let text_length = plain_text.chars().count() as i64;
                            total_chars += text_length;
                            total_block_count += 1;
                            cell_count += 1;
                            document_position += text_length;
                            if cell_count < total_cells {
                                document_position += 1;
                            }
                        }
                    }

                    // 3. Create anchor frame (links table to the flow)
                    let anchor_frame = Frame {
                        parent_frame: Some(current_frame_id),
                        table: Some(created_table.id),
                        ..Frame::default()
                    };
                    let created_anchor = uow.create_frame(&anchor_frame, doc_id, -1)?;

                    // Backfill each cell frame's `parent_frame` to point at
                    // the anchor frame. Cell frames are created before the
                    // anchor (we don't know the anchor id yet), so this
                    // can't be set at creation time. Without this, walking
                    // up from a cell to find its containing table fails —
                    // breaking `insert_table_uc`'s "is the cursor inside an
                    // existing table?" check.
                    for cell_frame_id in &created_cell_frame_ids {
                        if let Some(cf) = uow.get_frame(cell_frame_id)? {
                            let mut updated = cf;
                            updated.parent_frame = Some(created_anchor.id);
                            // `update_frame` is scalar-only and preserves
                            // the existing parent_frame to guard against
                            // stale-entity writes; use the relationship-
                            // aware variant so this write actually lands.
                            uow.update_frame_with_relationships(&updated)?;
                        }
                    }

                    frame_stack
                        .last_mut()
                        .unwrap()
                        .child_order
                        .push(-(created_anchor.id as i64));

                    // Tables put content in the rope (the anchor
                    // sentinel + cell blocks) — any subsequent
                    // main-flow block must be preceded by a `\n`
                    // boundary, same as after a Block.
                    emitted_any_main_block = true;

                    if i < total_elements - 1 {
                        document_position += 1;
                    }
                }
            }

            if i % 10 == 0 {
                let pct = 20.0 + (i as f32 / total_elements as f32) * 70.0;
                progress_callback(common::long_operation::OperationProgress::new(
                    pct,
                    Some(format!("Processing element {}/{}", i + 1, total_elements)),
                ));
            }
        }

        // Close any remaining open blockquote frames
        while frame_stack.len() > 1 {
            let finished = frame_stack.pop().unwrap();
            let mut frame_entity = uow
                .get_frame(&finished.frame_id)?
                .ok_or_else(|| anyhow!("Blockquote frame not found"))?;
            frame_entity.child_order = finished.child_order;
            uow.update_frame(&frame_entity)?;
        }

        // Step 5: Update root frame child_order
        let root_state = frame_stack.pop().unwrap();
        let mut updated_frame = uow
            .get_frame(&root_state.frame_id)?
            .ok_or_else(|| anyhow!("Created frame not found"))?;
        updated_frame.child_order = root_state.child_order;
        uow.update_frame(&updated_frame)?;

        // Step 6: Update document stats
        let mut updated_doc = uow
            .get_document(&doc_id)?
            .ok_or_else(|| anyhow!("Document not found after import"))?;
        updated_doc.character_count = total_chars;
        updated_doc.block_count = total_block_count;
        uow.update_document(&updated_doc)?;

        if cancel_flag.load(Ordering::Relaxed) {
            uow.rollback()?;
            return Err(anyhow!("Operation was cancelled"));
        }

        // Plan §1.6 Frame.byte_range maintenance happens in
        // Transaction::commit.
        uow.commit()?;

        progress_callback(common::long_operation::OperationProgress::new(
            100.0,
            Some("completed".to_string()),
        ));

        Ok(ImportHtmlResultDto {
            block_count: total_block_count,
        })
    }
}