use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
use crate::error::{Error, Result};
use crate::http::Fetcher;
use crate::parse::dom::{Doc, Id};
use crate::search::RawHit;
use crate::text;
pub(crate) async fn search(
fetcher: &Fetcher,
base: &str,
query: &str,
limit: usize,
) -> Result<Vec<RawHit>> {
let base = base.trim_end_matches('/');
crate::http::normalize(base)?;
let q = utf8_percent_encode(query, NON_ALPHANUMERIC);
if let Ok(page) = fetcher.fetch_api(&format!("{base}/search?q={q}&format=json")).await
&& let Ok(hits) = parse_json(&page.body, limit)
&& !hits.is_empty()
{
return Ok(hits);
}
let page = fetcher.fetch_api(&format!("{base}/search?q={q}")).await?;
let hits = parse_html(&page.body, limit);
if hits.is_empty() {
return Err(Error::provider(
"searxng",
format!("{base} returned neither JSON nor a parseable result list"),
));
}
Ok(hits)
}
pub(crate) fn parse_json(body: &str, limit: usize) -> Result<Vec<RawHit>> {
let value: serde_json::Value = serde_json::from_str(body)
.map_err(|e| Error::provider("searxng", format!("bad json: {e}")))?;
let results = value
.get("results")
.and_then(|r| r.as_array())
.ok_or_else(|| Error::provider("searxng", "response had no results array"))?;
Ok(results
.iter()
.filter_map(|r| {
let url = r.get("url")?.as_str()?.to_string();
let title = r.get("title").and_then(|t| t.as_str()).unwrap_or_default().to_string();
let snippet = r.get("content").and_then(|c| c.as_str()).unwrap_or_default().to_string();
Some(RawHit { title, url, snippet })
})
.take(limit)
.collect())
}
pub(crate) fn parse_html(html: &str, limit: usize) -> Vec<RawHit> {
let Ok(doc) = Doc::parse(html) else { return Vec::new() };
let mut out = Vec::new();
for &id in &doc.preorder {
if doc.tag_name(id) != "article" || !has_class(&doc, id, "result") {
continue;
}
let link = doc.descendants(id).into_iter().find(|&d| {
doc.tag_name(d) == "a"
&& has_class(&doc, d, "url_wrapper")
&& doc.attr(d, "href").is_some()
});
let heading = doc
.descendants(id)
.into_iter()
.find(|&d| doc.tag_name(d) == "h3")
.and_then(|h| doc.descendants(h).into_iter().find(|&d| doc.tag_name(d) == "a"));
let anchor = heading.or(link);
let Some(anchor) = anchor else { continue };
let Some(url) = doc.attr(anchor, "href").filter(|h| h.starts_with("http")) else {
continue;
};
let title = text::normalize_ws(&doc.inner_text(anchor));
let snippet = doc
.descendants(id)
.into_iter()
.find(|&d| has_class(&doc, d, "content"))
.map(|d| text::normalize_ws(&doc.inner_text(d)))
.unwrap_or_default();
out.push(RawHit { title, url, snippet });
if out.len() >= limit {
break;
}
}
out
}
fn has_class(doc: &Doc<'_>, id: Id, needle: &str) -> bool {
doc.attr(id, "class").is_some_and(|c| c.split_whitespace().any(|c| c == needle))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_the_json_api() {
let body = r#"{"results":[
{"url":"https://a.dev/1","title":"A","content":"first"},
{"url":"https://b.dev/2","title":"B","content":"second"}]}"#;
let hits = parse_json(body, 10).unwrap();
assert_eq!(hits.len(), 2);
assert_eq!(hits[0].url, "https://a.dev/1");
assert_eq!(hits[1].snippet, "second");
assert_eq!(parse_json(body, 1).unwrap().len(), 1);
}
#[test]
fn rejects_a_non_searxng_json_body() {
assert!(parse_json("{\"oops\":1}", 5).is_err());
assert!(parse_json("<html>", 5).is_err());
}
#[test]
fn parses_the_rendered_result_list() {
let html = r#"<html><body>
<article class="result result-default">
<a href="https://a.dev/1" class="url_wrapper"><span>a.dev</span></a>
<h3><a href="https://a.dev/1">First result</a></h3>
<p class="content">Snippet text here.</p>
</article>
<article class="result"><h3><a href="/local">skip me</a></h3></article>
</body></html>"#;
let hits = parse_html(html, 10);
assert_eq!(hits.len(), 1, "{hits:#?}");
assert_eq!(hits[0].url, "https://a.dev/1");
assert_eq!(hits[0].title, "First result");
assert_eq!(hits[0].snippet, "Snippet text here.");
}
}