html-to-markdown-rs 3.9.2

High-performance HTML to Markdown converter using the astral-tl parser. Part of the Xberg ecosystem.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

// ~keep Rust inner attributes below are crate-level attributes, not a shell shebang.
#![allow(missing_docs)]

//! Regression coverage for deeply nested and malformed markup. `tl` preserves
//! repeated unclosed `<td>` tags as a multi-thousand-level DOM chain, so these
//! tests run conversion on small thread stacks to catch native-stack recursion
//! in whole-subtree helpers.
//!
//! The `<td>` shape reaches hierarchy recording, metadata extraction, and table
//! scanning while the main conversion walker remains bounded by its own depth
//! guard. That keeps failures attributable to the helper traversal under test.

use html_to_markdown_rs::convert;
use html_to_markdown_rs::options::{ConversionOptions, OutputFormat};
use std::sync::{Mutex, MutexGuard};
use std::thread;

static TEST_MUTEX: Mutex<()> = Mutex::new(());

fn test_lock() -> MutexGuard<'static, ()> {
    TEST_MUTEX.lock().expect("deep nesting test mutex poisoned")
}

fn converts_without_overflow(html: String, options: ConversionOptions) -> bool {
    converts_without_overflow_on_stack(html, options, 256 * 1024)
}

fn converts_without_overflow_on_stack(html: String, options: ConversionOptions, stack_size: usize) -> bool {
    thread::Builder::new()
        .stack_size(stack_size)
        .spawn(move || convert(&html, Some(options)).is_ok())
        .expect("spawn conversion thread")
        .join()
        .expect("conversion thread overflowed the stack")
}

/// Exercises `record_node_hierarchy` (pre-pass) and `scan_table_node` (table
/// scan); the `<head>` is found without descending the deep chain.
#[test]
fn deep_unclosed_table_cells_do_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><head><title>t</title></head><body><table><tr>");
    for _ in 0..20_000 {
        html.push_str("<td>x");
    }
    html.push_str("</tr></table></body></html>");
    let options = ConversionOptions::builder().max_depth(Some(200)).build();
    assert!(converts_without_overflow(html, options));
}

/// No `<head>`, so metadata extraction must search the entire deep chain.
#[test]
fn deep_markup_without_head_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body><table><tr>");
    for _ in 0..20_000 {
        html.push_str("<td>x");
    }
    html.push_str("</tr></table></body></html>");
    let options = ConversionOptions::builder().max_depth(Some(200)).build();
    assert!(converts_without_overflow(html, options));
}

#[test]
fn deep_link_descendant_text_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><head><title>t</title></head><body><a href=\"/deep\">");
    for _ in 0..1_000 {
        html.push_str("<span>");
    }
    html.push_str("deep");
    html.push_str("</a></body></html>");

    let options = ConversionOptions::builder().max_depth(Some(200)).build();
    assert!(converts_without_overflow_on_stack(html, options, 8 * 1024 * 1024));
}

#[test]
fn default_depth_uses_stack_safe_limit() {
    let _guard = test_lock();
    let mut html = String::from("<html><body>");
    for _ in 0..1_000 {
        html.push_str("<div>");
    }
    html.push_str("deep");
    for _ in 0..1_000 {
        html.push_str("</div>");
    }
    html.push_str("</body></html>");

    assert!(converts_without_overflow(html, ConversionOptions::default()));
}

#[test]
fn plain_text_output_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body>");
    for _ in 0..1_000 {
        html.push_str("<div>");
    }
    html.push_str("deep");
    for _ in 0..1_000 {
        html.push_str("</div>");
    }
    html.push_str("</body></html>");

    let options = ConversionOptions {
        output_format: OutputFormat::Plain,
        max_depth: Some(200),
        ..Default::default()
    };
    // ~keep Plain output forces the recursive Tier-2 walk, and issue #434 lets an explicit
    // ~keep max_depth exceed the native stack-safe default (64), so depth 200 genuinely recurses
    // ~keep 200 frames. Raising the ceiling is the caller opting into deeper traversal, which needs
    // ~keep a proportionate stack — mirror the 8 MiB budget used by deep_link_descendant_text.
    assert!(converts_without_overflow_on_stack(html, options, 8 * 1024 * 1024));
}

#[test]
fn document_structure_builder_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body>");
    for _ in 0..1_000 {
        html.push_str("<section>");
    }
    html.push_str("<p>deep</p>");
    for _ in 0..1_000 {
        html.push_str("</section>");
    }
    html.push_str("</body></html>");

    assert!(
        thread::Builder::new()
            .stack_size(8 * 1024 * 1024)
            .spawn(move || {
                let dom = tl::parse(&html, tl::ParserOptions::default()).expect("parse deep html");
                let document = html_to_markdown_rs::types::build_document_structure(&dom);
                !document.nodes.is_empty()
            })
            .expect("spawn structure thread")
            .join()
            .expect("structure thread overflowed the stack")
    );
}