use microclaw_core::text::floor_char_boundary;
use std::borrow::Cow;
#[derive(Debug, Clone)]
pub struct SearchItem {
pub title: String,
pub url: String,
pub snippet: String,
}
fn strip_block(mut html: String, tag: &str) -> String {
let open = format!("<{}", tag);
let close = format!("</{}>", tag);
while let Some(start) = find_case_insensitive(&html, &open, 0) {
let Some(end) = find_case_insensitive(&html, &close, start) else {
html.truncate(start);
break;
};
let remove_to = end + close.len();
html.replace_range(start..remove_to, " ");
}
html
}
fn find_case_insensitive(haystack: &str, needle: &str, from: usize) -> Option<usize> {
if from >= haystack.len() {
return None;
}
let from = microclaw_core::text::floor_char_boundary(haystack, from);
let h = haystack[from..].to_ascii_lowercase();
let n = needle.to_ascii_lowercase();
h.find(&n).map(|idx| from + idx)
}
pub fn decode_html_entities(input: &str) -> Cow<'_, str> {
if !input.contains('&') {
return Cow::Borrowed(input);
}
let mut out = input.to_string();
let replacements = [
(" ", " "),
("&", "&"),
("<", "<"),
(">", ">"),
(""", "\""),
("'", "'"),
];
for (from, to) in replacements {
out = out.replace(from, to);
}
Cow::Owned(out)
}
pub fn html_to_text(html: &str) -> String {
let html = strip_block(strip_block(html.to_string(), "script"), "style");
let mut text = String::with_capacity(html.len());
let mut in_tag = false;
for ch in html.chars() {
match ch {
'<' => {
in_tag = true;
text.push(' ');
}
'>' => {
in_tag = false;
}
_ if !in_tag => text.push(ch),
_ => {}
}
}
collapse_whitespace(&decode_html_entities(&text))
}
pub fn collapse_whitespace(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let mut last_ws = false;
for ch in input.chars() {
if ch.is_whitespace() {
if !last_ws {
out.push(' ');
last_ws = true;
}
} else {
out.push(ch);
last_ws = false;
}
}
let mut compact = String::with_capacity(out.len());
let punctuation = ['.', ',', ':', ';', '!', '?'];
for ch in out.trim().chars() {
if punctuation.contains(&ch) && compact.ends_with(' ') {
compact.pop();
}
compact.push(ch);
}
compact
}
fn extract_attr(tag: &str, attr: &str) -> Option<String> {
let lower_tag = tag.to_ascii_lowercase();
let needle = format!("{}=", attr.to_ascii_lowercase());
let idx = lower_tag.find(&needle)?;
let raw = &tag[idx + needle.len()..];
let raw = raw.trim_start();
if raw.is_empty() {
return None;
}
let first = raw.chars().next()?;
if first == '"' || first == '\'' {
let quote = first;
let rest = &raw[1..];
let end = rest.find(quote)?;
return Some(rest[..end].to_string());
}
let end = raw
.find(|c: char| c.is_whitespace() || c == '>')
.unwrap_or(raw.len());
Some(raw[..end].to_string())
}
fn extract_snippet_near(html: &str, from: usize) -> String {
let from = floor_char_boundary(html, from.min(html.len()));
let window_end = floor_char_boundary(html, from.saturating_add(4000).min(html.len()));
let segment = &html[from..window_end];
let Some(class_idx) = find_case_insensitive(segment, "result__snippet", 0) else {
return String::new();
};
let tag_start = segment[..class_idx].rfind('<').unwrap_or(0);
let Some(tag_end_rel) = segment[tag_start..].find('>') else {
return String::new();
};
let tag_end = tag_start + tag_end_rel;
let closing = if segment[tag_start..].starts_with("<a") {
"</a>"
} else {
"</div>"
};
let Some(close_rel) = find_case_insensitive(segment, closing, tag_end) else {
return String::new();
};
let inner = &segment[tag_end + 1..close_rel];
html_to_text(inner)
}
pub fn extract_ddg_results(html: &str, max_results: usize) -> Vec<SearchItem> {
let mut results = Vec::new();
let mut pos = 0usize;
while results.len() < max_results {
let Some(a_start) = find_case_insensitive(html, "<a", pos) else {
break;
};
let Some(a_tag_end_rel) = html[a_start..].find('>') else {
break;
};
let a_tag_end = a_start + a_tag_end_rel;
let a_tag = &html[a_start..=a_tag_end];
let class = extract_attr(a_tag, "class").unwrap_or_default();
if !class.split_whitespace().any(|c| c == "result__a") {
pos = a_tag_end + 1;
continue;
}
let Some(close_rel) = find_case_insensitive(html, "</a>", a_tag_end + 1) else {
break;
};
let href = extract_attr(a_tag, "href")
.map(|h| decode_html_entities(&h).into_owned())
.unwrap_or_default();
let title_html = &html[a_tag_end + 1..close_rel];
let title = html_to_text(title_html);
let snippet = extract_snippet_near(html, close_rel + 4);
if !href.is_empty() && !title.is_empty() {
results.push(SearchItem {
title,
url: href,
snippet,
});
}
pos = close_rel + 4;
}
results
}
pub fn extract_primary_html(html: &str) -> &str {
let candidates = ["main", "article", "body"];
for tag in candidates {
let open = format!("<{tag}");
let close = format!("</{tag}>");
if let Some(start) = find_case_insensitive(html, &open, 0) {
if let Some(open_end_rel) = html[start..].find('>') {
let content_start = start + open_end_rel + 1;
if let Some(end) = find_case_insensitive(html, &close, content_start) {
return &html[content_start..end];
}
}
}
}
html
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_html_to_text() {
let html = "<html><body><h1>Hello World</h1><script>x=1;</script></body></html>";
assert_eq!(html_to_text(html), "Hello World");
}
#[test]
fn test_extract_ddg_results() {
let html = r#"
<div>
<a class="result__a" href="https://example.com">Example <b>Title</b></a>
<a class="result__snippet">This is <b>snippet</b>.</a>
</div>
"#;
let items = extract_ddg_results(html, 8);
assert_eq!(items.len(), 1);
assert_eq!(items[0].title, "Example Title");
assert_eq!(items[0].url, "https://example.com");
assert_eq!(items[0].snippet, "This is snippet.");
}
#[test]
fn test_extract_primary_html_prefers_main() {
let html = "<body>body</body><main>main section</main>";
assert_eq!(extract_primary_html(html), "main section");
}
#[test]
fn test_find_case_insensitive_non_char_boundary_input() {
let s = "abc只def";
assert_eq!(find_case_insensitive(s, "def", 4), Some(6));
}
}