mdream 1.3.0

Fastest HTML-to-Markdown converter. Zero dependencies, streaming support.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

pub mod consts;
pub(crate) mod convert;
pub(crate) mod entities;
pub(crate) mod scan;
pub(crate) mod selector;
pub mod splitter;
pub(crate) mod tags;
pub(crate) mod tailwind;
pub mod types;
pub(crate) mod url;

use convert::ConvertState;

// Re-export the public option/config types at the crate root so `use mdream::*`
// pulls in everything needed to call `html_to_markdown` without reaching into
// the `types` module.
pub use types::{
    CleanConfig, ExtractionConfig, FilterConfig, FrontmatterConfig, HTMLToMarkdownOptions,
    IsolateMainConfig, MdreamResult, PluginConfig, TagOverrideConfig, TailwindConfig,
};

// Re-export `get_tag_id` so callers can resolve tag names to IDs (for
// `TagOverrideConfig::alias_tag_id`) without reaching into `consts` directly.
pub use consts::get_tag_id;

/// Convert HTML to Markdown in a single pass.
pub fn html_to_markdown(html: &str, options: HTMLToMarkdownOptions) -> String {
    html_to_markdown_result(html, options).markdown
}

/// Convert HTML to Markdown with full results (extraction, frontmatter).
pub fn html_to_markdown_result(html: &str, options: HTMLToMarkdownOptions) -> MdreamResult {
    let capacity = (html.len() / 3).clamp(1024, 256 * 1024);
    let mut state = ConvertState::new(options, capacity);
    state.process_html(html);

    let extracted = if state.has_extraction {
        let results = std::mem::take(&mut state.extraction_results);
        if results.is_empty() { None } else { Some(results) }
    } else {
        None
    };

    let frontmatter = state.frontmatter();

    MdreamResult {
        markdown: state.get_markdown(),
        extracted,
        frontmatter,
    }
}

/// Streaming HTML-to-Markdown converter.
///
/// Feed chunks of HTML via `process_chunk()`, then call `finish()` for remaining output.
pub struct MarkdownStreamProcessor {
    state: ConvertState,
    buffer: String,
}

impl MarkdownStreamProcessor {
    pub fn new(options: HTMLToMarkdownOptions) -> Self {
        Self {
            state: ConvertState::new(options, 4096),
            buffer: String::new(),
        }
    }

    pub fn process_chunk(&mut self, chunk: &str) -> String {
        if self.buffer.is_empty() {
            self.buffer = self.state.process_html(chunk);
        } else {
            self.buffer.push_str(chunk);
            let full = std::mem::take(&mut self.buffer);
            self.buffer = self.state.process_html(&full);
        }
        self.state.get_markdown_chunk()
    }

    pub fn finish(&mut self) -> String {
        if !self.buffer.is_empty() {
            let chunk = std::mem::take(&mut self.buffer);
            self.state.process_html(&chunk);
        }
        self.state.get_markdown_chunk()
    }
}