web2md 0.1.3

A tool that fetches web pages and returns them as Markdown for MCP token efficiency
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use std::time::Duration;

mod browser;
mod crawl;
mod html_meta;
mod html_to_md;
mod html_util;
mod js;
mod markdown;
mod mcp;
mod robots;
mod url_blacklist;

pub use browser::{extract_feed_links, parse_sitemap_urls, Browser, BrowserOptions};
pub use crawl::{normalize_crawl_url, same_origin_links};
pub use markdown::PageToMarkdown;
pub use mcp::{extract_metadata, McpRequest, McpServer};

const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
const DEFAULT_USER_AGENT: &str = concat!(
    "Web2MD/",
    env!("CARGO_PKG_VERSION"),
    " (Web to Markdown Converter; +https://github.com/yingkitw/web2md)"
);