html-to-markdown-rs 3.11.0

High-performance HTML to Markdown converter using the astral-tl parser. Part of the Xberg ecosystem.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249

// ~keep Rust inner attributes below are crate-level attributes, not a shell shebang.
#![allow(missing_docs)]

//! Regression coverage for deeply nested and malformed markup. `tl` preserves
//! repeated unclosed `<td>` tags as a multi-thousand-level DOM chain, so these
//! tests run conversion on small thread stacks to catch native-stack recursion
//! in whole-subtree helpers.
//!
//! The `<td>` shape reaches hierarchy recording, metadata extraction, and table
//! scanning while the main conversion walker remains bounded by its own depth
//! guard. That keeps failures attributable to the helper traversal under test.

use html_to_markdown_rs::ConversionResult;
use html_to_markdown_rs::convert;
use html_to_markdown_rs::options::{ConversionOptions, OutputFormat};
use std::sync::{Mutex, MutexGuard};
use std::thread;

static TEST_MUTEX: Mutex<()> = Mutex::new(());

fn test_lock() -> MutexGuard<'static, ()> {
    TEST_MUTEX.lock().expect("deep nesting test mutex poisoned")
}

fn converts_without_overflow(html: String, options: ConversionOptions) -> bool {
    converts_without_overflow_on_stack(html, options, 256 * 1024)
}

fn converts_without_overflow_on_stack(html: String, options: ConversionOptions, stack_size: usize) -> bool {
    thread::Builder::new()
        .stack_size(stack_size)
        .spawn(move || convert(&html, Some(options)).is_ok())
        .expect("spawn conversion thread")
        .join()
        .expect("conversion thread overflowed the stack")
}

/// Like [`converts_without_overflow`] but returns the successful result so callers can
/// inspect the (possibly truncated) output instead of only checking `Result::is_ok`.
fn convert_without_overflow_on_stack(html: String, options: ConversionOptions, stack_size: usize) -> ConversionResult {
    thread::Builder::new()
        .stack_size(stack_size)
        .spawn(move || convert(&html, Some(options)))
        .expect("spawn conversion thread")
        .join()
        .expect("conversion thread overflowed the stack")
        .expect("conversion should not fail")
}

/// Exercises `record_node_hierarchy` (pre-pass) and `scan_table_node` (table
/// scan); the `<head>` is found without descending the deep chain.
#[test]
fn deep_unclosed_table_cells_do_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><head><title>t</title></head><body><table><tr>");
    for _ in 0..20_000 {
        html.push_str("<td>x");
    }
    html.push_str("</tr></table></body></html>");
    let options = ConversionOptions::builder().max_depth(Some(200)).build();
    assert!(converts_without_overflow(html, options));
}

/// No `<head>`, so metadata extraction must search the entire deep chain.
#[test]
fn deep_markup_without_head_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body><table><tr>");
    for _ in 0..20_000 {
        html.push_str("<td>x");
    }
    html.push_str("</tr></table></body></html>");
    let options = ConversionOptions::builder().max_depth(Some(200)).build();
    assert!(converts_without_overflow(html, options));
}

#[test]
fn deep_link_descendant_text_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><head><title>t</title></head><body><a href=\"/deep\">");
    for _ in 0..1_000 {
        html.push_str("<span>");
    }
    html.push_str("deep");
    html.push_str("</a></body></html>");

    let options = ConversionOptions::builder().max_depth(Some(200)).build();
    assert!(converts_without_overflow_on_stack(html, options, 8 * 1024 * 1024));
}

#[test]
fn default_depth_uses_stack_safe_limit() {
    let _guard = test_lock();
    let mut html = String::from("<html><body>");
    for _ in 0..1_000 {
        html.push_str("<div>");
    }
    html.push_str("deep");
    for _ in 0..1_000 {
        html.push_str("</div>");
    }
    html.push_str("</body></html>");

    assert!(converts_without_overflow(html, ConversionOptions::default()));
}

#[test]
fn plain_text_output_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body>");
    for _ in 0..1_000 {
        html.push_str("<div>");
    }
    html.push_str("deep");
    for _ in 0..1_000 {
        html.push_str("</div>");
    }
    html.push_str("</body></html>");

    let options = ConversionOptions {
        output_format: OutputFormat::Plain,
        max_depth: Some(200),
        ..Default::default()
    };
    // ~keep Plain output forces the recursive Tier-2 walk, and issue #434 lets an explicit
    // ~keep max_depth exceed the native stack-safe default (64), so depth 200 genuinely recurses
    // ~keep 200 frames. Raising the ceiling is the caller opting into deeper traversal, which needs
    // ~keep a proportionate stack — mirror the 8 MiB budget used by deep_link_descendant_text.
    assert!(converts_without_overflow_on_stack(html, options, 8 * 1024 * 1024));
}

/// Exercises `media::svg::serialize_element_at_depth`/`serialize_node_at_depth`
/// (audit #23): the SVG-to-base64 embedding path used to mutually recurse over
/// `<g>` children with no depth bound at all, independent of the main walker's
/// own depth guard (the `<svg>` element is serialized as a single leaf, so its
/// internal children never pass through `walk_node`).
#[test]
fn deep_svg_group_nesting_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body><svg>");
    for _ in 0..50_000 {
        html.push_str("<g>");
    }
    for _ in 0..50_000 {
        html.push_str("</g>");
    }
    html.push_str("</svg></body></html>");

    let result = convert_without_overflow_on_stack(html, ConversionOptions::default(), 256 * 1024);
    let content = result.content.as_deref().unwrap_or_default();
    assert!(
        content.contains("data:image/svg+xml;base64,"),
        "expected a (truncated) base64 SVG data URI in the output. Got:\n{content}"
    );
}

/// Same `serialize_element_at_depth`/`serialize_node_at_depth` path as
/// `deep_svg_group_nesting_does_not_overflow_stack`, exercised via `<math>`/`<mrow>`
/// instead of `<svg>`/`<g>` since `handle_math` shares the same serializer.
#[test]
fn deep_mathml_nesting_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body><math>");
    for _ in 0..50_000 {
        html.push_str("<mrow>");
    }
    html.push_str("<mi>x</mi>");
    for _ in 0..50_000 {
        html.push_str("</mrow>");
    }
    html.push_str("</math></body></html>");

    let result = convert_without_overflow_on_stack(html, ConversionOptions::default(), 256 * 1024);
    let content = result.content.as_deref().unwrap_or_default();
    assert!(
        content.contains("<!-- MathML: "),
        "expected a (truncated) MathML comment in the output. Got:\n{content}"
    );
}

/// Regression coverage: table-cell conversion used to reset the `walk_node` recursion
/// depth to a literal `0` at every cell boundary (`table/cell.rs`, `table/cells.rs`,
/// `table/mod.rs`, `table/builder.rs`), instead of threading the real depth through as
/// `depth + 1` like every other descent in `converter/`. Because each `<table>` nested
/// inside a `<td>` restarted the counter, `walk_node`'s own depth guard
/// (`crate::converter::main::effective_max_depth`) could never fire for a chain of
/// nested tables, so a deeply nested `<table><tr><td>` chain recursed without bound and
/// could overflow the native stack. The fix threads `depth + 1` through every cell/row
/// descent so the guard now covers nested tables the same way it covers every other
/// element chain.
#[test]
fn deeply_nested_tables_do_not_overflow_stack() {
    // ~keep 2,000 is ~30x the native-stack-safe depth guard (64), which is enough to prove
    // ~keep the guard fires reliably; `scan_table_node` (table/scanner.rs) independently walks
    // ~keep the full remaining nested-table chain from every table's own position (a separate,
    // ~keep pre-existing O(n^2) cost unrelated to this depth-reset fix), so a much larger depth
    // ~keep here would make this test impractically slow without adding overflow coverage.
    const NESTING_DEPTH: usize = 2_000;

    let _guard = test_lock();

    let mut html = String::from("<html><body>");
    for _ in 0..NESTING_DEPTH {
        html.push_str("<table><tr><td>");
    }
    html.push_str("leaf");
    for _ in 0..NESTING_DEPTH {
        html.push_str("</td></tr></table>");
    }
    html.push_str("</body></html>");

    let result = convert_without_overflow_on_stack(html, ConversionOptions::default(), 256 * 1024);
    let content = result.content.as_deref().unwrap_or_default();
    assert!(
        !content.trim().is_empty(),
        "deeply nested table conversion must not silently return empty output"
    );
    assert!(
        content.contains('|'),
        "expected at least the outer table rows to render as pipe-delimited content. Got:\n{content}"
    );
}

#[test]
fn document_structure_builder_does_not_overflow_stack() {
    let _guard = test_lock();
    let mut html = String::from("<html><body>");
    for _ in 0..1_000 {
        html.push_str("<section>");
    }
    html.push_str("<p>deep</p>");
    for _ in 0..1_000 {
        html.push_str("</section>");
    }
    html.push_str("</body></html>");

    assert!(
        thread::Builder::new()
            .stack_size(8 * 1024 * 1024)
            .spawn(move || {
                let dom = tl::parse(&html, tl::ParserOptions::default()).expect("parse deep html");
                let document = html_to_markdown_rs::types::build_document_structure(&dom);
                !document.nodes.is_empty()
            })
            .expect("spawn structure thread")
            .join()
            .expect("structure thread overflowed the stack")
    );
}