use std::collections::HashSet;
use std::sync::Arc;
use async_trait::async_trait;
use crate::application::dto::map_request::MapRequestDto;
use crate::engines::engine_client::EngineClient;
use crate::engines::types::ScrapeRequest;
pub struct SitemapFetch {
pub status: u16,
pub body: String,
}
#[derive(Debug, thiserror::Error)]
pub enum MapError {
#[error("target unreachable: {0}")]
TargetUnreachable(String),
#[error("sitemap returned status {0}")]
SitemapStatus(u16),
#[error("cannot parse origin from {0}")]
OriginUnparsable(String),
#[error("internal: {0}")]
Internal(String),
}
#[cfg(feature = "sitemap")]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SitemapKind {
Urlset,
SitemapIndex,
}
#[cfg(feature = "sitemap")]
pub struct SitemapStream {
pub kind: SitemapKind,
pub rx: tokio::sync::mpsc::Receiver<String>,
}
#[cfg(feature = "sitemap")]
impl SitemapStream {
pub fn empty() -> Self {
let (tx, rx) = tokio::sync::mpsc::channel(1);
drop(tx);
Self {
kind: SitemapKind::Urlset,
rx,
}
}
}
#[cfg(feature = "sitemap")]
#[derive(Debug, Clone)]
pub struct SitemapStreamOptions {
pub channel_capacity: usize,
pub max_urls: usize,
pub max_concurrent_sub_sitemaps: usize,
#[doc(hidden)]
pub domain_blacklist: Vec<String>,
}
#[async_trait]
pub trait SitemapFetcher: Send + Sync {
async fn fetch(&self, url: &str) -> Result<SitemapFetch, MapError>;
#[cfg(feature = "sitemap")]
async fn fetch_streaming(
&self,
url: &str,
channel_capacity: usize,
) -> Result<SitemapStream, MapError> {
let fetched = self.fetch(url).await?;
if let Some(verdict) = classify_sitemap_status(fetched.status) {
return verdict;
}
let kind = if is_sitemap_index(&fetched.body) {
SitemapKind::SitemapIndex
} else {
SitemapKind::Urlset
};
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
tokio::spawn(async move {
for loc in parse_locs(&fetched.body) {
if tx.send(loc).await.is_err() {
break;
}
}
});
Ok(SitemapStream { kind, rx })
}
}
pub struct EngineSitemapFetcher {
client: Arc<EngineClient>,
timeout: std::time::Duration,
}
impl EngineSitemapFetcher {
pub fn new(client: Arc<EngineClient>, timeout: std::time::Duration) -> Self {
Self { client, timeout }
}
}
#[async_trait]
impl SitemapFetcher for EngineSitemapFetcher {
async fn fetch(&self, url: &str) -> Result<SitemapFetch, MapError> {
let request = ScrapeRequest::new(url.to_string()).timeout(self.timeout);
let response = self
.client
.scrape(&request)
.await
.map_err(|e| MapError::TargetUnreachable(e.to_string()))?;
Ok(SitemapFetch {
status: response.status_code,
body: response.content,
})
}
#[cfg(feature = "sitemap")]
async fn fetch_streaming(
&self,
url: &str,
channel_capacity: usize,
) -> Result<SitemapStream, MapError> {
let request = ScrapeRequest::new(url.to_string()).timeout(self.timeout);
let response = self
.client
.scrape(&request)
.await
.map_err(|e| MapError::TargetUnreachable(e.to_string()))?;
if let Some(verdict) = classify_sitemap_status(response.status_code) {
return verdict;
}
Ok(spawn_loc_stream(response.content, channel_capacity))
}
}
#[cfg(feature = "sitemap")]
fn classify_sitemap_status(status: u16) -> Option<Result<SitemapStream, MapError>> {
if status == 404 {
return Some(Ok(SitemapStream::empty()));
}
if status >= 400 {
return Some(Err(MapError::SitemapStatus(status)));
}
if status != 200 {
return Some(Ok(SitemapStream::empty()));
}
None
}
#[cfg(feature = "sitemap")]
fn spawn_loc_stream(xml: String, channel_capacity: usize) -> SitemapStream {
let kind = if is_sitemap_index(&xml) {
SitemapKind::SitemapIndex
} else {
SitemapKind::Urlset
};
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
tokio::spawn(stream_loc_events(xml, tx));
SitemapStream { kind, rx }
}
#[cfg(feature = "sitemap")]
async fn stream_loc_events(xml: String, tx: tokio::sync::mpsc::Sender<String>) {
use quick_xml::events::Event;
let mut reader = quick_xml::Reader::from_str(&xml);
let mut in_loc = false;
let mut loc_text = String::new();
loop {
match reader.read_event() {
Ok(Event::Start(e)) => {
if is_loc_name(e.name().local_name().as_ref()) {
in_loc = true;
loc_text.clear();
}
}
Ok(Event::Text(t)) if in_loc => push_loc_text(&mut loc_text, &t),
Ok(Event::End(e)) => {
if in_loc && is_loc_name(e.name().local_name().as_ref()) {
in_loc = false;
if emit_completed_loc(&tx, &loc_text).await {
break;
}
}
}
Ok(Event::Eof) => break,
Ok(_) => {}
Err(e) => {
log::warn!(
"sitemap XML parse error: {e}; keeping partial results already produced"
);
break;
}
}
}
}
#[cfg(feature = "sitemap")]
fn is_loc_name(local_name: &str) -> bool {
local_name.eq_ignore_ascii_case("loc")
}
#[cfg(feature = "sitemap")]
fn push_loc_text(loc_text: &mut String, t: &quick_xml::events::BytesText<'_>) {
match quick_xml::escape::unescape(&t.xml10_content()) {
Ok(unescaped) => loc_text.push_str(&unescaped),
Err(e) => {
log::warn!("sitemap loc entity unescape failed: {e}");
}
}
}
#[cfg(feature = "sitemap")]
async fn emit_completed_loc(tx: &tokio::sync::mpsc::Sender<String>, loc_text: &str) -> bool {
let loc = loc_text.trim();
!loc.is_empty() && tx.send(loc.to_string()).await.is_err()
}
pub struct MapResult {
pub links: Vec<String>,
}
pub struct MapUseCase {
fetcher: Arc<dyn SitemapFetcher>,
}
const MAX_SUB_SITEMAPS: usize = 5;
impl MapUseCase {
pub fn new(fetcher: Arc<dyn SitemapFetcher>) -> Self {
Self { fetcher }
}
pub async fn execute(&self, dto: &MapRequestDto) -> Result<MapResult, MapError> {
let origin =
extract_origin(&dto.url).ok_or_else(|| MapError::OriginUnparsable(dto.url.clone()))?;
let first = self.fetcher.fetch(&format!("{origin}/sitemap.xml")).await?;
if first.status == 404 {
return Ok(MapResult { links: Vec::new() });
}
if first.status >= 400 {
return Err(MapError::SitemapStatus(first.status));
}
if first.status != 200 {
return Ok(MapResult { links: Vec::new() });
}
let mut locs = parse_locs(&first.body);
if is_sitemap_index(&first.body) {
locs = self.expand_sitemap_index(locs).await?;
}
Ok(MapResult {
links: finalize_links(locs, dto),
})
}
async fn expand_sitemap_index(&self, mut locs: Vec<String>) -> Result<Vec<String>, MapError> {
let children: Vec<String> = locs.iter().take(MAX_SUB_SITEMAPS).cloned().collect();
locs.clear();
for child in children {
let fetched = match self.fetcher.fetch(&child).await {
Ok(f) => Some(f),
Err(MapError::TargetUnreachable(_)) | Err(MapError::SitemapStatus(_)) => {
self.fetcher.fetch(&child).await.ok()
}
Err(e) => return Err(e),
};
if let Some(fetched) = fetched {
if fetched.status == 200 {
locs.extend(parse_locs(&fetched.body));
}
}
}
Ok(locs)
}
#[cfg(feature = "sitemap")]
pub async fn execute_streaming(
&self,
dto: &MapRequestDto,
options: SitemapStreamOptions,
) -> Result<SitemapStream, MapError> {
let origin =
extract_origin(&dto.url).ok_or_else(|| MapError::OriginUnparsable(dto.url.clone()))?;
let mut first = self
.fetcher
.fetch_streaming(&format!("{origin}/sitemap.xml"), options.channel_capacity)
.await?;
match first.kind {
SitemapKind::Urlset => Ok(SitemapStream {
kind: SitemapKind::Urlset,
rx: forward_capped(first.rx, options),
}),
SitemapKind::SitemapIndex => {
let children = collect_index_children(&mut first.rx).await;
Ok(self.expand_index_stream(children, options))
}
}
}
#[cfg(feature = "sitemap")]
fn expand_index_stream(
&self,
children: Vec<String>,
options: SitemapStreamOptions,
) -> SitemapStream {
use tokio::sync::mpsc;
use tokio::task::JoinSet;
let (tx, rx) = mpsc::channel(options.channel_capacity.max(1));
let semaphore = Arc::new(tokio::sync::Semaphore::new(
options.max_concurrent_sub_sitemaps.max(1),
));
let sent = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut set: JoinSet<()> = JoinSet::new();
for child in children {
if let Some(reason) =
crate::domain::services::team_service::domain_in_normalized_denylist(
&child,
&options.domain_blacklist,
)
{
log::warn!("skip blacklisted child sitemap: {reason}");
continue;
}
let fetcher = self.fetcher.clone();
let tx = tx.clone();
let semaphore = semaphore.clone();
let sent = sent.clone();
set.spawn(forward_child_sitemap(
fetcher,
child,
options.clone(),
tx,
semaphore,
sent,
));
}
drop(tx);
spawn_child_task_monitor(set);
SitemapStream {
kind: SitemapKind::SitemapIndex,
rx,
}
}
}
#[cfg(feature = "sitemap")]
async fn forward_child_sitemap(
fetcher: Arc<dyn SitemapFetcher>,
child: String,
options: SitemapStreamOptions,
tx: tokio::sync::mpsc::Sender<String>,
semaphore: Arc<tokio::sync::Semaphore>,
sent: Arc<std::sync::atomic::AtomicUsize>,
) {
let _permit = semaphore
.acquire_owned()
.await
.expect("semaphore has capacity");
let Some(stream) =
fetch_child_stream_with_retry(&fetcher, &child, options.channel_capacity).await
else {
return;
};
let mut rx = stream.rx;
while let Some(loc) = rx.recv().await {
if sent.load(std::sync::atomic::Ordering::Relaxed) >= options.max_urls {
return;
}
if tx.send(loc).await.is_err() {
return;
}
sent.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
}
#[cfg(feature = "sitemap")]
async fn fetch_child_stream_with_retry(
fetcher: &Arc<dyn SitemapFetcher>,
child: &str,
channel_capacity: usize,
) -> Option<SitemapStream> {
match fetcher.fetch_streaming(child, channel_capacity).await {
Ok(s) => Some(s),
Err(MapError::TargetUnreachable(_)) | Err(MapError::SitemapStatus(_)) => {
fetcher.fetch_streaming(child, channel_capacity).await.ok()
}
Err(_) => None,
}
}
#[cfg(feature = "sitemap")]
fn spawn_child_task_monitor(mut set: tokio::task::JoinSet<()>) {
tokio::spawn(async move {
while let Some(res) = set.join_next().await {
if let Err(e) = res {
log::warn!("sitemap child task failed: {e}");
}
}
});
}
#[cfg(feature = "sitemap")]
async fn collect_index_children(rx: &mut tokio::sync::mpsc::Receiver<String>) -> Vec<String> {
let mut children: Vec<String> = Vec::new();
while children.len() < MAX_SUB_SITEMAPS {
match rx.recv().await {
Some(child) => children.push(child),
None => break,
}
}
children
}
#[cfg(feature = "sitemap")]
fn forward_capped(
mut rx: tokio::sync::mpsc::Receiver<String>,
options: SitemapStreamOptions,
) -> tokio::sync::mpsc::Receiver<String> {
let (tx, out) = tokio::sync::mpsc::channel(options.channel_capacity.max(1));
tokio::spawn(async move {
let mut sent = 0usize;
while sent < options.max_urls {
match rx.recv().await {
Some(loc) => {
if tx.send(loc).await.is_err() {
break;
}
sent += 1;
}
None => break,
}
}
});
out
}
pub fn finalize_links(locs: Vec<String>, dto: &MapRequestDto) -> Vec<String> {
let mut links = locs;
if let Some(include) = &dto.include_patterns {
links.retain(|u| include.iter().any(|p| matches_glob(p, u)));
}
if let Some(exclude) = &dto.exclude_patterns {
links.retain(|u| !exclude.iter().any(|p| matches_glob(p, u)));
}
let mut seen: HashSet<String> = HashSet::with_capacity(links.len());
links.retain(|u| seen.insert(u.clone()));
let limit = dto.limit.unwrap_or(1000) as usize;
links.truncate(limit);
links
}
fn extract_origin(url: &str) -> Option<String> {
let parsed = url::Url::parse(url).ok()?;
let host = parsed.host_str()?;
let scheme = parsed.scheme();
match parsed.port() {
Some(port) => Some(format!("{scheme}://{host}:{port}")),
None => Some(format!("{scheme}://{host}")),
}
}
pub fn parse_locs(xml: &str) -> Vec<String> {
let document = scraper::Html::parse_fragment(xml);
document
.select(&scraper::Selector::parse("loc").expect("valid selector"))
.map(|el| el.text().collect::<String>().trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
fn is_sitemap_index(xml: &str) -> bool {
xml.to_ascii_lowercase().contains("<sitemapindex")
}
pub fn matches_glob(pattern: &str, text: &str) -> bool {
crate::utils::glob::glob_match(pattern, text)
}
#[cfg(test)]
mod tests {
use super::*;
use async_trait::async_trait;
use std::collections::HashMap;
struct MockFetcher {
responses: HashMap<String, (u16, String)>,
}
#[async_trait]
impl SitemapFetcher for MockFetcher {
async fn fetch(&self, url: &str) -> Result<SitemapFetch, MapError> {
match self.responses.get(url) {
Some((status, body)) => Ok(SitemapFetch {
status: *status,
body: body.clone(),
}),
None => Err(MapError::TargetUnreachable(format!("no mock for {url}"))),
}
}
}
fn urlset(locs: &[&str]) -> String {
let inner: String = locs
.iter()
.map(|l| format!("<url><loc>{l}</loc></url>"))
.collect();
format!(
r#"<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{inner}</urlset>"#
)
}
fn sitemap_index(children: &[&str]) -> String {
let inner: String = children
.iter()
.map(|l| format!("<sitemap><loc>{l}</loc></sitemap>"))
.collect();
format!(
r#"<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{inner}</sitemapindex>"#
)
}
fn dto(url: &str) -> MapRequestDto {
MapRequestDto {
url: url.to_string(),
include_patterns: None,
exclude_patterns: None,
limit: Some(1000),
}
}
const ORIGIN: &str = "https://a.com";
#[test]
fn parse_locs_preserves_order() {
let xml = urlset(&["https://a.com/1", "https://a.com/2", "https://a.com/3"]);
assert_eq!(
parse_locs(&xml),
vec!["https://a.com/1", "https://a.com/2", "https://a.com/3"]
);
}
#[test]
fn parse_locs_empty_sitemap() {
assert!(parse_locs(&urlset(&[])).is_empty());
}
#[test]
fn matches_glob_semantics() {
assert!(matches_glob("*/blog/*", "https://a.com/blog/post-1"));
assert!(!matches_glob("*/blog/*", "https://a.com/page/1"));
assert!(matches_glob("https://a.com/page/?", "https://a.com/page/1"));
assert!(!matches_glob(
"https://a.com/page/?",
"https://a.com/page/12"
));
assert!(matches_glob("*", "https://anything.example"));
assert!(matches_glob("https://a.com/*", "https://a.com/"));
}
#[tokio::test]
async fn execute_returns_locs_in_order() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(
200u16,
urlset(&["https://a.com/1", "https://a.com/2", "https://a.com/3"]),
),
)]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await
.expect("execute");
assert_eq!(
result.links,
vec!["https://a.com/1", "https://a.com/2", "https://a.com/3"]
);
}
#[tokio::test]
async fn execute_dedups_keeping_first_position() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(
200u16,
urlset(&["https://a.com/a", "https://a.com/b", "https://a.com/a"]),
),
)]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await
.expect("execute");
assert_eq!(result.links, vec!["https://a.com/a", "https://a.com/b"]);
}
#[tokio::test]
async fn execute_recurses_sitemap_index_one_level() {
let fetcher = MockFetcher {
responses: HashMap::from([
(
format!("{ORIGIN}/sitemap.xml"),
(
200u16,
sitemap_index(&["https://a.com/s1.xml", "https://a.com/s2.xml"]),
),
),
(
"https://a.com/s1.xml".to_string(),
(
200u16,
urlset(&["https://a.com/1", "https://a.com/2", "https://a.com/3"]),
),
),
(
"https://a.com/s2.xml".to_string(),
(200u16, urlset(&["https://a.com/4", "https://a.com/5"])),
),
]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await
.expect("execute");
assert_eq!(result.links.len(), 5);
assert_eq!(result.links[0], "https://a.com/1");
assert_eq!(result.links[4], "https://a.com/5");
}
#[tokio::test]
async fn execute_caps_sub_sitemaps_at_five() {
let children: Vec<String> = (0..7).map(|i| format!("https://a.com/s{i}.xml")).collect();
let child_refs: Vec<&str> = children.iter().map(|s| s.as_str()).collect();
let mut responses = HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(200u16, sitemap_index(&child_refs)),
)]);
for i in 0..7 {
responses.insert(
format!("https://a.com/s{i}.xml"),
(200u16, urlset(&[&format!("https://a.com/u{i}")])),
);
}
let fetcher = MockFetcher { responses };
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await
.expect("execute");
assert_eq!(
result.links.len(),
5,
"only first 5 sub sitemaps must be fetched"
);
assert!(result.links.contains(&"https://a.com/u0".to_string()));
assert!(!result.links.contains(&"https://a.com/u5".to_string()));
}
#[tokio::test]
async fn execute_applies_include_then_exclude_patterns() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(
200u16,
urlset(&[
"https://a.com/blog/1",
"https://a.com/blog/tag/x",
"https://a.com/page/1",
]),
),
)]),
};
let mut d = dto(ORIGIN);
d.include_patterns = Some(vec!["*/blog/*".to_string()]);
d.exclude_patterns = Some(vec!["*/tag/*".to_string()]);
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&d)
.await
.expect("execute");
assert_eq!(result.links, vec!["https://a.com/blog/1"]);
}
#[tokio::test]
async fn execute_truncates_to_limit() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(
200u16,
urlset(&["https://a.com/1", "https://a.com/2", "https://a.com/3"]),
),
)]),
};
let mut d = dto(ORIGIN);
d.limit = Some(2);
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&d)
.await
.expect("execute");
assert_eq!(result.links, vec!["https://a.com/1", "https://a.com/2"]);
}
#[tokio::test]
async fn execute_returns_empty_links_on_sitemap_404() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(404u16, String::from("not found")),
)]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await
.expect("execute");
assert!(result.links.is_empty());
}
#[tokio::test]
async fn execute_non_200_redirect_returns_empty_links() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(
302u16,
urlset(&["https://a.com/redirected-should-not-leak"]),
),
)]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await
.expect("3xx must be Ok (empty links)");
assert!(
result.links.is_empty(),
"redirect body must not be parsed as sitemap"
);
}
#[tokio::test]
async fn execute_maps_5xx_to_target_unreachable() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(500u16, String::from("boom")),
)]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto(ORIGIN))
.await;
assert!(matches!(result, Err(MapError::SitemapStatus(_))));
}
#[tokio::test]
async fn execute_rejects_unparsable_url() {
let fetcher = MockFetcher {
responses: HashMap::new(),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute(&dto("not-a-url"))
.await;
assert!(matches!(result, Err(MapError::OriginUnparsable(_))));
}
#[cfg(feature = "sitemap")]
mod streaming_tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn stream_options(
capacity: usize,
max_urls: usize,
concurrency: usize,
) -> SitemapStreamOptions {
SitemapStreamOptions {
domain_blacklist: Vec::new(),
channel_capacity: capacity,
max_urls,
max_concurrent_sub_sitemaps: concurrency,
}
}
async fn collect(rx: tokio::sync::mpsc::Receiver<String>) -> Vec<String> {
let mut rx = rx;
let mut out = Vec::new();
while let Some(loc) = rx.recv().await {
out.push(loc);
}
out
}
#[tokio::test]
async fn spawn_loc_stream_does_not_block_when_loc_count_exceeds_capacity() {
let locs: Vec<String> = (0..8).map(|i| format!("https://a.com/{i}")).collect();
let refs: Vec<&str> = locs.iter().map(String::as_str).collect();
let stream = spawn_loc_stream(urlset(&refs), 1);
assert_eq!(stream.kind, SitemapKind::Urlset);
let received =
tokio::time::timeout(std::time::Duration::from_secs(5), collect(stream.rx))
.await
.expect("producer must not deadlock when loc count exceeds capacity");
assert_eq!(received, locs);
}
#[tokio::test]
async fn streaming_urlset_yields_ten_thousand_locs_in_order() {
let locs: Vec<String> = (0..10_000).map(|i| format!("https://a.com/{i}")).collect();
let refs: Vec<&str> = locs.iter().map(String::as_str).collect();
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(200u16, urlset(&refs)),
)]),
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(64, 50_000, 4))
.await
.expect("execute_streaming");
assert_eq!(collect(rx.rx).await, locs, "万级 loc 必须逐条且保序产出");
}
#[tokio::test]
async fn streaming_sitemap_404_yields_empty_stream() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(404u16, String::from("not found")),
)]),
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 100, 2))
.await
.expect("404 must be Ok (empty stream)");
assert!(collect(rx.rx).await.is_empty());
}
#[tokio::test]
async fn streaming_non_200_redirect_yields_empty_stream() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(
302u16,
urlset(&["https://a.com/redirected-should-not-leak"]),
),
)]),
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 100, 2))
.await
.expect("3xx must be Ok (empty stream)");
assert!(
collect(rx.rx).await.is_empty(),
"redirect body must not be parsed as sitemap"
);
}
#[tokio::test]
async fn streaming_maps_5xx_to_target_unreachable() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(500u16, String::from("boom")),
)]),
};
let result = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 100, 2))
.await;
assert!(matches!(result, Err(MapError::SitemapStatus(_))));
}
#[tokio::test]
async fn streaming_respects_max_urls_cap() {
let locs: Vec<String> = (0..100).map(|i| format!("https://a.com/{i}")).collect();
let refs: Vec<&str> = locs.iter().map(String::as_str).collect();
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(200u16, urlset(&refs)),
)]),
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 10, 2))
.await
.expect("execute_streaming");
let got = collect(rx.rx).await;
assert_eq!(got.len(), 10, "max_urls=10 must cap the stream at 10");
assert_eq!(got[0], "https://a.com/0");
assert_eq!(got[9], "https://a.com/9");
}
struct BackpressureFetcher {
total: usize,
sent: Arc<AtomicUsize>,
}
#[async_trait]
impl SitemapFetcher for BackpressureFetcher {
async fn fetch(&self, _url: &str) -> Result<SitemapFetch, MapError> {
Err(MapError::Internal("unused".to_string()))
}
async fn fetch_streaming(
&self,
_url: &str,
channel_capacity: usize,
) -> Result<SitemapStream, MapError> {
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
let sent = self.sent.clone();
let total = self.total;
tokio::spawn(async move {
for i in 0..total {
if tx.send(format!("https://a.com/{i}")).await.is_err() {
break;
}
sent.fetch_add(1, Ordering::SeqCst);
}
});
Ok(SitemapStream {
kind: SitemapKind::Urlset,
rx,
})
}
}
#[tokio::test]
async fn streaming_backpressure_bounds_producer_progress() {
const TOTAL: usize = 10_000;
const CAPACITY: usize = 4;
let sent = Arc::new(AtomicUsize::new(0));
let fetcher = BackpressureFetcher {
total: TOTAL,
sent: sent.clone(),
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(CAPACITY, TOTAL, 2))
.await
.expect("execute_streaming");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
let stalled = sent.load(Ordering::SeqCst);
assert!(
stalled <= 2 * CAPACITY + 1,
"consumer paused: producer must be bounded by channel capacity, \
sent={stalled} (cap={CAPACITY})"
);
let got = collect(rx.rx).await;
assert_eq!(got.len(), TOTAL, "drain must yield all locs");
assert_eq!(
sent.load(Ordering::SeqCst),
TOTAL,
"producer must complete after drain"
);
}
struct IndexMockFetcher {
children: HashMap<String, (u16, String)>,
root_children: Vec<String>,
in_flight: Arc<AtomicUsize>,
max_in_flight: Arc<AtomicUsize>,
child_delay_ms: u64,
}
impl IndexMockFetcher {
fn record_enter(&self) {
let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1;
self.max_in_flight.fetch_max(now, Ordering::SeqCst);
}
fn root_stream(&self, channel_capacity: usize) -> SitemapStream {
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
let children = self.root_children.clone();
tokio::spawn(async move {
for child in children {
if tx.send(child).await.is_err() {
break;
}
}
});
SitemapStream {
kind: SitemapKind::SitemapIndex,
rx,
}
}
async fn child_stream(
&self,
url: &str,
channel_capacity: usize,
) -> Result<SitemapStream, MapError> {
let (status, body) = match self.children.get(url) {
Some((s, b)) => (*s, b.clone()),
None => return Err(MapError::TargetUnreachable(format!("no mock for {url}"))),
};
if status == 404 {
return Ok(SitemapStream::empty());
}
if status >= 400 {
return Err(MapError::TargetUnreachable(format!(
"child returned status {status}"
)));
}
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
self.record_enter();
let in_flight = self.in_flight.clone();
let delay = self.child_delay_ms;
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(delay)).await;
for loc in parse_locs(&body) {
if tx.send(loc).await.is_err() {
break;
}
}
in_flight.fetch_sub(1, Ordering::SeqCst);
});
Ok(SitemapStream {
kind: SitemapKind::Urlset,
rx,
})
}
}
#[async_trait]
impl SitemapFetcher for IndexMockFetcher {
async fn fetch(&self, _url: &str) -> Result<SitemapFetch, MapError> {
Err(MapError::Internal("unused".to_string()))
}
async fn fetch_streaming(
&self,
url: &str,
channel_capacity: usize,
) -> Result<SitemapStream, MapError> {
const ROOT: &str = "https://a.com/sitemap.xml";
if url == ROOT {
return Ok(self.root_stream(channel_capacity));
}
self.child_stream(url, channel_capacity).await
}
}
#[tokio::test]
async fn streaming_index_recursion_caps_concurrency_and_children() {
let mut children = HashMap::new();
let mut root_children = Vec::new();
for i in 0..7 {
let url = format!("https://a.com/s{i}.xml");
let status = if i == 1 { 500u16 } else { 200u16 };
let body = urlset(&[&format!("https://a.com/u{i}")]);
children.insert(url.clone(), (status, body));
root_children.push(url);
}
let in_flight = Arc::new(AtomicUsize::new(0));
let max_in_flight = Arc::new(AtomicUsize::new(0));
let fetcher = IndexMockFetcher {
children,
root_children,
in_flight: in_flight.clone(),
max_in_flight: max_in_flight.clone(),
child_delay_ms: 50,
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 100, 2))
.await
.expect("execute_streaming");
let got = collect(rx.rx).await;
assert_eq!(
got,
vec![
"https://a.com/u0",
"https://a.com/u2",
"https://a.com/u3",
"https://a.com/u4",
],
"first 5 children expanded, failed child skipped"
);
let peak = max_in_flight.load(Ordering::SeqCst);
assert!(
peak >= 2,
"children must be fetched concurrently (observed peak {peak})"
);
assert!(
peak <= 2,
"child fetch concurrency must not exceed max_concurrent_sub_sitemaps=2, got {peak}"
);
}
#[tokio::test]
async fn streaming_empty_index_yields_empty_stream() {
let fetcher = MockFetcher {
responses: HashMap::from([(
format!("{ORIGIN}/sitemap.xml"),
(200u16, sitemap_index(&[])),
)]),
};
let rx = MapUseCase::new(Arc::new(fetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 100, 2))
.await
.expect("execute_streaming");
assert!(collect(rx.rx).await.is_empty());
}
struct PanicChildFetcher;
#[async_trait]
impl SitemapFetcher for PanicChildFetcher {
async fn fetch(&self, _url: &str) -> Result<SitemapFetch, MapError> {
Err(MapError::Internal("unused".to_string()))
}
async fn fetch_streaming(
&self,
url: &str,
channel_capacity: usize,
) -> Result<SitemapStream, MapError> {
const ROOT: &str = "https://a.com/sitemap.xml";
if url == ROOT {
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
let children = [
"https://a.com/panic.xml".to_string(),
"https://a.com/ok.xml".to_string(),
];
tokio::spawn(async move {
for child in children {
if tx.send(child).await.is_err() {
break;
}
}
});
return Ok(SitemapStream {
kind: SitemapKind::SitemapIndex,
rx,
});
}
if url.ends_with("/panic.xml") {
panic!("mock child panic");
}
let (tx, rx) = tokio::sync::mpsc::channel(channel_capacity.max(1));
tokio::spawn(async move {
let _ = tx.send("https://a.com/u-ok".to_string()).await;
});
Ok(SitemapStream {
kind: SitemapKind::Urlset,
rx,
})
}
}
#[tokio::test]
async fn streaming_child_panic_does_not_break_stream() {
let rx = MapUseCase::new(Arc::new(PanicChildFetcher))
.execute_streaming(&dto(ORIGIN), stream_options(4, 100, 2))
.await
.expect("execute_streaming");
let got = collect(rx.rx).await;
assert_eq!(
got,
vec!["https://a.com/u-ok".to_string()],
"healthy child must still deliver after a sibling panics"
);
}
#[test]
fn finalize_links_matches_execute_semantics() {
let mut d = dto(ORIGIN);
d.include_patterns = Some(vec!["*/blog/*".to_string()]);
d.exclude_patterns = Some(vec!["*/tag/*".to_string()]);
d.limit = Some(2);
let links = finalize_links(
vec![
"https://a.com/blog/1".to_string(),
"https://a.com/blog/1".to_string(), "https://a.com/blog/tag/x".to_string(), "https://a.com/page/1".to_string(), "https://a.com/blog/2".to_string(),
"https://a.com/blog/3".to_string(),
],
&d,
);
assert_eq!(links, vec!["https://a.com/blog/1", "https://a.com/blog/2"]);
}
}
}