use super::*;
#[tokio::test]
async fn test_is_allowed_with_cached_robots_allows_non_disallowed_path() {
let checker = make_checker();
let content = "User-agent: *\nDisallow: /private\n";
populate_robots_cache(&checker, "https://example.com:443/robots.txt", content).await;
let allowed = checker
.is_allowed("https://example.com/page", "MyBot/1.0")
.await
.expect("should succeed with cached content");
assert!(allowed, "non-disallowed path should be allowed");
}
#[tokio::test]
async fn test_is_allowed_with_cached_robots_blocks_disallowed_path() {
let checker = make_checker();
let content = "User-agent: *\nDisallow: /private\n";
populate_robots_cache(&checker, "https://example.com:443/robots.txt", content).await;
let allowed = checker
.is_allowed("https://example.com/private", "MyBot/1.0")
.await
.expect("should succeed with cached content");
assert!(!allowed, "disallowed path should be blocked");
}
#[tokio::test]
async fn test_is_allowed_with_cached_robots_blocks_disallowed_subpath() {
let checker = make_checker();
let content = "User-agent: *\nDisallow: /private\n";
populate_robots_cache(&checker, "https://example.com:443/robots.txt", content).await;
let allowed = checker
.is_allowed("https://example.com/private/secret", "MyBot/1.0")
.await
.expect("should succeed with cached content");
assert!(
!allowed,
"subpath of disallowed path should also be blocked"
);
}
#[tokio::test]
async fn test_is_allowed_with_cached_empty_robots_allows_all() {
let checker = make_checker();
populate_robots_cache(&checker, "https://example.com:443/robots.txt", "").await;
let allowed = checker
.is_allowed("https://example.com/anything", "MyBot/1.0")
.await
.expect("should succeed with empty robots.txt");
assert!(allowed, "empty robots.txt should allow all paths");
}
#[tokio::test]
async fn test_is_allowed_with_cached_robots_specific_agent_block() {
let checker = make_checker();
let content = "User-agent: BadBot\nDisallow: /\n";
populate_robots_cache(&checker, "https://example.com:443/robots.txt", content).await;
let bad_allowed = checker
.is_allowed("https://example.com/page", "BadBot")
.await
.expect("should succeed");
assert!(!bad_allowed, "BadBot should be blocked on all paths");
let other_content = "User-agent: BadBot\nDisallow: /\nUser-agent: *\nAllow: /\n";
populate_robots_cache(
&checker,
"https://example.com:443/robots.txt",
other_content,
)
.await;
let good_allowed = checker
.is_allowed("https://example.com/page", "GoodBot")
.await
.expect("should succeed");
assert!(good_allowed, "GoodBot should be allowed via * group");
}
#[tokio::test]
async fn test_get_crawl_delay_with_cached_robots() {
let checker = make_checker();
let content = "User-agent: *\nCrawl-delay: 5\n";
populate_robots_cache(&checker, "https://example.com:443/robots.txt", content).await;
let delay = checker
.get_crawl_delay("https://example.com/page", "MyBot/1.0")
.await
.expect("should succeed with cached content");
assert_eq!(
delay,
Some(Duration::from_secs(5)),
"should return the cached crawl delay"
);
}
#[tokio::test]
async fn test_get_crawl_delay_with_cached_no_delay_directive() {
let checker = make_checker();
let content = "User-agent: *\nDisallow: /private\n";
populate_robots_cache(&checker, "https://example.com:443/robots.txt", content).await;
let delay = checker
.get_crawl_delay("https://example.com/page", "MyBot/1.0")
.await
.expect("should succeed");
assert_eq!(
delay, None,
"should return None when no Crawl-delay directive"
);
}
#[tokio::test]
async fn test_is_allowed_cache_hit_increments_stats() {
let http_client = Arc::new(reqwest::Client::new());
let stats = Arc::new(CacheStats::default());
let checker = RobotsChecker::new(
http_client,
&crate::config::settings::RobotsSettings::default(),
None,
Some(stats.clone()),
);
populate_robots_cache(
&checker,
"https://example.com:443/robots.txt",
"User-agent: *\nDisallow: /private\n",
)
.await;
let (hits_before, misses_before) = checker.get_cache_stats();
assert_eq!(hits_before, 0);
assert_eq!(misses_before, 0);
checker
.is_allowed("https://example.com/page", "MyBot")
.await
.expect("should succeed");
let (hits_after, misses_after) = checker.get_cache_stats();
assert_eq!(hits_after, 1, "cache hit should increment hits");
assert_eq!(misses_after, 0, "no misses should occur for cache hit");
}
#[tokio::test]
async fn test_get_crawl_delay_cache_hit_increments_stats() {
let http_client = Arc::new(reqwest::Client::new());
let stats = Arc::new(CacheStats::default());
let checker = RobotsChecker::new(
http_client,
&crate::config::settings::RobotsSettings::default(),
None,
Some(stats.clone()),
);
populate_robots_cache(
&checker,
"https://example.com:443/robots.txt",
"User-agent: *\nCrawl-delay: 3\n",
)
.await;
checker
.get_crawl_delay("https://example.com/page", "MyBot")
.await
.expect("should succeed");
let (hits, misses) = checker.get_cache_stats();
assert_eq!(hits, 1, "cache hit should be recorded");
assert_eq!(misses, 0);
}
#[tokio::test]
async fn test_is_allowed_invalid_url_returns_error() {
let checker = make_checker();
let result = checker.is_allowed("not-a-valid-url", "MyBot").await;
assert!(result.is_err(), "invalid URL should return an error");
}
#[tokio::test]
async fn test_is_allowed_url_without_host_returns_error() {
let checker = make_checker();
let result = checker.is_allowed("file:///etc/passwd", "MyBot").await;
assert!(result.is_err(), "URL without host should return an error");
}
#[tokio::test]
async fn test_get_crawl_delay_invalid_url_returns_error() {
let checker = make_checker();
let result = checker.get_crawl_delay("not-a-valid-url", "MyBot").await;
assert!(result.is_err(), "invalid URL should return an error");
}
#[tokio::test]
async fn test_get_crawl_delay_url_without_host_returns_error() {
let checker = make_checker();
let result = checker.get_crawl_delay("file:///etc/passwd", "MyBot").await;
assert!(result.is_err(), "URL without host should return an error");
}
#[test]
fn test_new_uses_settings_user_agent() {
let http_client = Arc::new(reqwest::Client::new());
let settings = crate::config::settings::RobotsSettings {
user_agent: "crawlrs-bot/test".to_string(),
..Default::default()
};
let checker = RobotsChecker::new(http_client, &settings, None, None);
assert_eq!(checker.user_agent(), "crawlrs-bot/test");
}