mdream/
lib.rs

1pub mod consts;
2pub(crate) mod convert;
3pub(crate) mod entities;
4pub(crate) mod scan;
5pub(crate) mod selector;
6pub mod splitter;
7pub(crate) mod tags;
8pub(crate) mod tailwind;
9pub mod types;
10pub(crate) mod url;
11
12use convert::ConvertState;
13
14// Re-export the public option/config types at the crate root so `use mdream::*`
15// pulls in everything needed to call `html_to_markdown` without reaching into
16// the `types` module.
17pub use types::{
18    CleanConfig, ExtractionConfig, FilterConfig, FrontmatterConfig, HTMLToMarkdownOptions,
19    IsolateMainConfig, MdreamResult, PluginConfig, TagOverrideConfig, TailwindConfig,
20};
21
22// Re-export `get_tag_id` so callers can resolve tag names to IDs (for
23// `TagOverrideConfig::alias_tag_id`) without reaching into `consts` directly.
24pub use consts::get_tag_id;
25
26/// Convert HTML to Markdown in a single pass.
27pub fn html_to_markdown(html: &str, options: HTMLToMarkdownOptions) -> String {
28    html_to_markdown_result(html, options).markdown
29}
30
31/// Convert HTML to Markdown with full results (extraction, frontmatter).
32pub fn html_to_markdown_result(html: &str, options: HTMLToMarkdownOptions) -> MdreamResult {
33    let capacity = (html.len() / 3).clamp(1024, 256 * 1024);
34    let mut state = ConvertState::new(options, capacity);
35    let leftover = state.process_html(html);
36    state.finalize(&leftover);
37
38    let extracted = if state.has_extraction {
39        let results = std::mem::take(&mut state.extraction_results);
40        if results.is_empty() { None } else { Some(results) }
41    } else {
42        None
43    };
44
45    let frontmatter = state.frontmatter();
46
47    MdreamResult {
48        markdown: state.get_markdown(),
49        extracted,
50        frontmatter,
51    }
52}
53
54/// Streaming HTML-to-Markdown converter.
55///
56/// Feed chunks of HTML via `process_chunk()`, then call `finish()` for remaining output.
57pub struct MarkdownStreamProcessor {
58    state: ConvertState,
59    buffer: String,
60}
61
62impl MarkdownStreamProcessor {
63    pub fn new(options: HTMLToMarkdownOptions) -> Self {
64        Self {
65            state: ConvertState::new(options, 4096),
66            buffer: String::new(),
67        }
68    }
69
70    pub fn process_chunk(&mut self, chunk: &str) -> String {
71        if self.buffer.is_empty() {
72            self.buffer = self.state.process_html(chunk);
73        } else {
74            self.buffer.push_str(chunk);
75            let full = std::mem::take(&mut self.buffer);
76            self.buffer = self.state.process_html(&full);
77        }
78        self.state.get_markdown_chunk()
79    }
80
81    pub fn finish(&mut self) -> String {
82        let leftover = if self.buffer.is_empty() {
83            String::new()
84        } else {
85            let chunk = std::mem::take(&mut self.buffer);
86            self.state.process_html(&chunk)
87        };
88        self.state.finalize(&leftover);
89        self.state.get_markdown_chunk()
90    }
91}
mdream/lib.rs

mdream/
lib.rs
