html-to-markdown-rs 3.6.21

High-performance HTML to Markdown converter using the astral-tl parser. Part of the Kreuzberg ecosystem.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396

#![allow(missing_docs)]

use html_to_markdown_rs::ConversionOptions;

#[test]
fn test_strip_simple_script_tag() {
    // Simple script with plain JavaScript
    let html = r"<html>
<head>
  <script>var x = 1; var y = 2;</script>
</head>
<body>
  <p>Real content here</p>
</body>
</html>";

    let options = ConversionOptions::default();
    let result = convert(html, Some(options)).expect("Failed to convert");

    println!("Output:\n{result}");

    // Should extract body content
    assert!(result.contains("Real content"), "Should contain body content");

    // Script content should not appear in output
    assert!(!result.contains("var x"), "Script content should be removed");
    assert!(!result.contains("var y"), "Script content should be removed");
}

#[test]
fn test_strip_script_with_html_like_content() {
    // Critical test: script with HTML-like content that confuses parsers
    let html = r#"<html>
<head>
  <script>
    var data = {
      html: '<div class="fake">This looks like HTML</div>',
      tags: '<span>test</span>',
      close: '</body>'
    };
  </script>
</head>
<body>
  <div id="real-content">
    <p>Real article content here</p>
  </div>
</body>
</html>"#;

    let options = ConversionOptions::default();
    let result = convert(html, Some(options)).expect("Failed to convert");

    println!("Output:\n{result}");

    // Should extract body content
    assert!(
        result.contains("Real article content"),
        "Should contain real body content"
    );

    // Script content should NOT appear
    assert!(!result.contains("fake"), "Fake HTML from script should be removed");
    assert!(!result.contains("var data"), "Script variables should not appear");
}

#[test]
fn test_strip_style_tag() {
    // Style tags with CSS that has HTML-like syntax
    let html = r"<html>
<head>
  <style>
    div { content: '<fake>test</fake>'; }
    body { display: block; }
  </style>
</head>
<body>
  <p>Paragraph content</p>
</body>
</html>";

    let options = ConversionOptions::default();
    let result = convert(html, Some(options)).expect("Failed to convert");

    println!("Output:\n{result}");

    // Should have body content
    assert!(result.contains("Paragraph content"), "Should contain paragraph");

    // Style content should be removed
    assert!(!result.contains("content:"), "CSS should not appear in output");
    assert!(!result.contains("display:"), "CSS properties should not appear");
}

#[test]
fn test_preserve_json_ld_script() {
    // JSON-LD script tags should be preserved for metadata extraction
    let html = r#"<html>
<head>
  <title>Article Title</title>
  <script type="application/ld+json">
    {
      "@context": "https://schema.org",
      "@type": "Article",
      "headline": "My Article",
      "author": { "@type": "Person", "name": "John Doe" }
    }
  </script>
</head>
<body>
  <p>Article content</p>
</body>
</html>"#;

    let result = html_to_markdown_rs::convert(html, None).expect("Failed to convert");
    let metadata = result.metadata;
    let markdown = result.content.unwrap_or_default();

    println!("Markdown:\n{markdown}");
    println!("Metadata: {:?}", metadata.document.title);

    // Should extract both content and metadata
    assert!(markdown.contains("Article content"), "Should contain body content");

    // Should have extracted metadata
    assert_eq!(
        metadata.document.title,
        Some("Article Title".to_string()),
        "Should extract title"
    );

    // JSON-LD should be in structured data
    assert!(!metadata.structured_data.is_empty(), "Should extract JSON-LD");
    if let Some(schema) = metadata.structured_data.first() {
        assert!(
            schema.raw_json.contains("Article"),
            "JSON-LD should contain Article type"
        );
        assert_eq!(
            schema.schema_type,
            Some("Article".to_string()),
            "Should detect schema type"
        );
    }
}

#[test]
fn test_multiple_script_tags() {
    // Multiple script tags with various content
    let html = r#"<html>
<head>
  <script>console.log('script 1');</script>
  <script type="text/javascript">
    if (x < y) {
      document.write('<p>Fake paragraph</p>');
    }
  </script>
  <script type="application/ld+json">
    {"@type": "WebPage", "@context": "https://schema.org"}
  </script>
</head>
<body>
  <h1>Real Title</h1>
  <p>Real paragraph</p>
</body>
</html>"#;

    let result = html_to_markdown_rs::convert(html, None).expect("Failed to convert");
    let metadata = result.metadata;
    let markdown = result.content.unwrap_or_default();

    println!("Markdown:\n{markdown}");

    // Should have real content
    assert!(markdown.contains("Real Title"), "Should have h1");
    assert!(markdown.contains("Real paragraph"), "Should have paragraph");

    // Should NOT have fake content from scripts
    assert!(
        !markdown.contains("Fake paragraph"),
        "Should not have fake HTML from script"
    );
    assert!(!markdown.contains("console.log"), "Should not have console.log");
    assert!(!markdown.contains("document.write"), "Should not have document.write");

    // JSON-LD should be extracted in metadata
    assert!(
        !metadata.structured_data.is_empty(),
        "Should extract JSON-LD structured data"
    );
}

#[test]
fn test_reuters_like_structure() {
    // Mimics Reuters structure with divs containing data-testid attributes
    let html = r#"<!DOCTYPE html>
<html>
<head>
  <title>Reuters Article</title>
  <meta property="og:title" content="Breaking News">
  <meta property="og:description" content="Important story">
  <script>
    window.data = {
      paragraphs: [
        '<div data-testid="paragraph-0">Fake content from script</div>',
        '<div data-testid="paragraph-1">Another fake</div>'
      ]
    };
  </script>
</head>
<body>
<article>
  <div data-testid="ArticleBody" class="article-body-module__wrapper">
    <div data-testid="paragraph-0" class="article-body-module__paragraph">
      SAN FRANCISCO, Dec 27 (Reuters) - A widespread power outage in San Francisco.
    </div>
    <div data-testid="paragraph-1" class="article-body-module__paragraph">
      The outage affected thousands of residents and businesses across the city.
    </div>
  </div>
</article>
</body>
</html>"#;

    let result = html_to_markdown_rs::convert(html, None).expect("Failed to convert");
    let metadata = result.metadata;
    let markdown = result.content.unwrap_or_default();

    println!("Markdown output:\n{markdown}");
    println!("Metadata title: {:?}", metadata.document.title);

    // Should have extracted metadata
    assert_eq!(
        metadata.document.title,
        Some("Reuters Article".to_string()),
        "Should extract title"
    );
    assert!(
        metadata.document.open_graph.contains_key("title"),
        "Should extract OG title"
    );

    // Should have article content
    assert!(markdown.contains("SAN FRANCISCO"), "Should contain first paragraph");
    assert!(
        markdown.contains("widespread power outage"),
        "Should contain article text"
    );
    assert!(
        markdown.contains("thousands of residents"),
        "Should contain second paragraph"
    );

    // Should NOT have fake content from script
    assert!(!markdown.contains("window.data"), "Should not have window.data");
    assert!(
        !markdown.contains("'<div data-testid"),
        "Should not have fake HTML strings from script"
    );
}

#[test]
fn test_complex_nested_script_content() {
    // Script with complex nested structures that might confuse parsers
    let html = r#"<html>
<head>
  <script>
    var config = {
      template: `
        <html>
          <body>
            <div class="container">
              <p>Nested HTML in template string</p>
            </div>
          </body>
        </html>
      `,
      patterns: [
        { regex: '/<body>.*?<\/body>/gs' },
        { html: '<script>alert("xss")</script>' }
      ]
    };
  </script>
</head>
<body>
  <section>
    <h2>Main Content</h2>
    <p>This is the actual article.</p>
  </section>
</body>
</html>"#;

    let options = ConversionOptions::default();
    let result = convert(html, Some(options)).expect("Failed to convert");

    println!("Output:\n{result}");

    // Should have actual content
    assert!(result.contains("Main Content"), "Should have h2");
    assert!(result.contains("actual article"), "Should have paragraph");

    // Should NOT have nested fake HTML from script
    assert!(
        !result.contains("Nested HTML in template"),
        "Should not have template content"
    );
    assert!(!result.contains("Container"), "Should not have nested div");
}

#[test]
fn test_case_insensitive_script_style_tags() {
    // Test that script/style tags are matched case-insensitively
    let html = r"<html>
<head>
  <SCRIPT>console.log('uppercase script');</SCRIPT>
  <Style>body { margin: 0; }</Style>
  <ScRiPt>var x = 1;</ScRiPt>
</head>
<body>
  <p>Content</p>
</body>
</html>";

    let options = ConversionOptions::default();
    let result = convert(html, Some(options)).expect("Failed to convert");

    println!("Output:\n{result}");

    // Should have content
    assert!(result.contains("Content"), "Should have content");

    // Should not have script content regardless of case
    assert!(!result.contains("console.log"), "Should remove SCRIPT tag content");
    assert!(!result.contains("margin:"), "Should remove STYLE tag content");
    assert!(!result.contains("var x"), "Should remove ScRiPt tag content");
}

#[test]
fn test_performance_large_script() {
    // Performance test: large script tag shouldn't cause issues
    let mut html = String::from(r"<html><head><script>");

    // Add 1MB of fake content
    for _ in 0..10000 {
        html.push_str("var data = '<div>fake content</div>'; ");
    }

    html.push_str(r"</script></head><body><p>Real content</p></body></html>");

    println!("Testing with {} byte HTML", html.len());

    let options = ConversionOptions::default();
    let result = convert(&html, Some(options)).expect("Failed to convert");

    // Should work and extract real content
    assert!(
        result.contains("Real content"),
        "Should extract body despite large script"
    );
}

#[test]
fn test_inline_script_attributes_not_affected() {
    // Inline event handlers should not be affected by script stripping
    let html = r#"<html>
<head>
  <script>console.log('bad');</script>
</head>
<body>
  <button onclick="console.log('click');">Click me</button>
  <p>Content</p>
</body>
</html>"#;

    let options = ConversionOptions::default();
    let result = convert(html, Some(options)).expect("Failed to convert");

    println!("Output:\n{result}");

    // Should have content and button
    assert!(result.contains("Click me"), "Should have button text");
    assert!(result.contains("Content"), "Should have paragraph");

    // The onclick script inside the tag attribute might appear in output (that's okay)
    // but the inline script tag content should be gone
    assert!(
        !result.contains("console.log('bad')"),
        "Should remove script tag content"
    );
}

fn convert(
    html: &str,
    opts: Option<html_to_markdown_rs::ConversionOptions>,
) -> html_to_markdown_rs::error::Result<String> {
    html_to_markdown_rs::convert(html, opts).map(|r| r.content.unwrap_or_default())
}