use super::*;
use crate::error::FetchError;
use crate::fetch::{FetchRequest, FetchResponse, Fetcher};
use async_trait::async_trait;
use std::collections::HashMap;
use std::sync::Mutex;
struct MockFetcher(String);
#[async_trait]
impl Fetcher for MockFetcher {
async fn fetch(&self, _req: FetchRequest) -> std::result::Result<String, FetchError> {
Ok(self.0.clone())
}
}
struct RecordingFetcher {
body: String,
last_headers: Arc<Mutex<HashMap<String, String>>>,
}
#[async_trait]
impl Fetcher for RecordingFetcher {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
*self.last_headers.lock().unwrap() = req.headers;
Ok(self.body.clone())
}
}
struct CookieEchoFetcher {
set_cookie: String,
last_cookie: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl Fetcher for CookieEchoFetcher {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
self.fetch_full(req).await.map(|r| r.body)
}
async fn fetch_full(
&self,
req: FetchRequest,
) -> std::result::Result<FetchResponse, FetchError> {
*self.last_cookie.lock().unwrap() = req.headers.get("Cookie").cloned();
let mut headers = HashMap::new();
headers.insert("set-cookie".to_string(), self.set_cookie.clone());
Ok(FetchResponse {
body: CATALOG.to_string(),
status: 200,
headers,
dom_html: None,
})
}
}
const CATALOG: &str = r#"<html><body><div class="box">
<span id="shuqian"><h2 class="module-title type">阅读进度</h2></span>
<h2 class="module-title type">第一卷</h2>
<div class="module-row-info"><a class="module-row-text" href="/n/1.html"><div class="module-row-title"><span>第一章</span></div></a></div>
<div class="module-row-info"><a class="module-row-text" href="/n/2.html"><div class="module-row-title"><span>第二章</span></div></a></div>
<h2 class="module-title type">第二卷</h2>
<div class="module-row-info"><a class="module-row-text" href="/n/3.html"><div class="module-row-title"><span>第三章</span></div></a></div>
</div></body></html>"#;
const SOURCE: &str = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"bookInfo":{},
"toc":{
"list":{"via":"css","select":".box > h2.module-title.type, .box a.module-row-text"},
"name":{"firstOf":[{"via":"css","select":".module-row-title","extract":"text"},{"via":"css","select":"h2","extract":"text"}]},
"url":{"via":"css","select":"a","extract":{"attr":"href"}},
"isVolume":{"via":"css","select":"h2","extract":"text"},
"maxPages":1
},
"content":{"value":{"via":"css","select":".article-content","extract":"text"}}
}"#;
#[tokio::test]
async fn engine_toc_splits_volumes_offline() {
let src = BookSource::from_json(SOURCE).unwrap();
let engine = Engine::with_fetcher(src, Arc::new(MockFetcher(CATALOG.to_string())));
let toc = engine.toc("/any").await.unwrap();
assert_eq!(toc.volumes.len(), 2, "应识别 2 卷");
assert_eq!(toc.chapters.len(), 3, "应识别 3 章");
assert_eq!(toc.chapters[0].title, "第一章");
assert_eq!(toc.chapters[0].url, "/n/1.html");
assert_eq!(toc.volumes[1].first_chapter_index, 2);
}
#[tokio::test]
async fn engine_merges_login_header_into_requests() {
let src = BookSource::from_json(SOURCE).unwrap();
let captured = Arc::new(Mutex::new(HashMap::new()));
let fetcher = Arc::new(RecordingFetcher {
body: CATALOG.to_string(),
last_headers: captured.clone(),
});
let mut lh = BTreeMap::new();
lh.insert("Authorization".into(), "Bearer T".into());
lh.insert("Cookie".into(), "sid=1".into());
let engine = Engine::with_fetcher(src, fetcher).with_login_header(lh);
engine.toc("/any").await.unwrap();
let h = captured.lock().unwrap();
assert_eq!(
h.get("Authorization").map(String::as_str),
Some("Bearer T"),
"JWT 应每请求携带"
);
assert_eq!(
h.get("Cookie").map(String::as_str),
Some("sid=1"),
"Cookie 走同一注入路径"
);
}
#[tokio::test]
async fn login_header_not_sent_to_other_registrable_domain() {
let src = BookSource::from_json(SOURCE).unwrap(); let captured = Arc::new(Mutex::new(HashMap::new()));
let fetcher = Arc::new(RecordingFetcher {
body: CATALOG.to_string(),
last_headers: captured.clone(),
});
let mut lh = BTreeMap::new();
lh.insert("Authorization".into(), "Bearer T".into());
lh.insert("Cookie".into(), "sid=1".into());
let engine = Engine::with_fetcher(src, fetcher).with_login_header(lh);
engine.toc("https://evil.example.org/any").await.unwrap();
let h = captured.lock().unwrap();
assert!(
h.get("Authorization").is_none(),
"跨注册域不应携带登录头: {h:?}"
);
assert!(
h.get("Cookie").is_none(),
"跨注册域不应携带登录 Cookie: {h:?}"
);
}
#[tokio::test]
async fn newline_in_login_header_sanitized_in_engine_requests() {
let src = BookSource::from_json(SOURCE).unwrap();
let captured = Arc::new(Mutex::new(HashMap::new()));
let fetcher = Arc::new(RecordingFetcher {
body: CATALOG.to_string(),
last_headers: captured.clone(),
});
let mut lh = BTreeMap::new();
lh.insert("Cookie".into(), "a=1\nb=2".into());
let engine = Engine::with_fetcher(src, fetcher).with_login_header(lh);
engine.toc("/any").await.unwrap();
let h = captured.lock().unwrap();
let cookie = h.get("Cookie").cloned().unwrap_or_default();
assert!(!cookie.contains('\n'), "Cookie 的 \\n 应被剥除: {cookie:?}");
assert_eq!(cookie, "a=1b=2", "与 host 侧 sanitize 行为对称");
}
#[tokio::test]
async fn engine_without_login_header_adds_nothing() {
let src = BookSource::from_json(SOURCE).unwrap();
let captured = Arc::new(Mutex::new(HashMap::new()));
let fetcher = Arc::new(RecordingFetcher {
body: CATALOG.to_string(),
last_headers: captured.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
engine.toc("/any").await.unwrap();
assert!(captured.lock().unwrap().is_empty(), "未登录不应注入额外头");
}
#[cfg(feature = "js")]
#[tokio::test]
async fn login_check_js_detects_expired() {
let json = SOURCE.replacen(
"\"bookInfo\":{}",
"\"loginCheckJs\":\"result.indexOf('未登录')<0\",\"bookInfo\":{}",
1,
);
let src = BookSource::from_json(&json).unwrap();
let bad = Engine::with_fetcher(
src.clone(),
Arc::new(MockFetcher("<html>未登录</html>".into())),
);
let err = bad.toc("/any").await.unwrap_err();
assert!(err.is_login_expired(), "应判登录失效: {err}");
let ok = Engine::with_fetcher(src, Arc::new(MockFetcher(CATALOG.to_string())));
assert!(ok.toc("/any").await.is_ok(), "正常响应不应判失效");
}
#[tokio::test]
async fn enabled_cookie_jar_absorbs_resends_and_persists() {
let json = SOURCE.replacen(
"\"bookInfo\":{}",
"\"enabledCookieJar\":true,\"bookInfo\":{}",
1,
);
let src = BookSource::from_json(&json).unwrap();
let last = Arc::new(Mutex::new(None));
let fetcher = Arc::new(CookieEchoFetcher {
set_cookie: "token=xyz; Max-Age=3600; Path=/".to_string(),
last_cookie: last.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
engine.toc("/p1").await.unwrap();
assert!(last.lock().unwrap().is_none(), "首请求不应带 cookie");
engine.book_info("/p2").await.unwrap();
assert_eq!(
last.lock().unwrap().clone(),
Some("token=xyz".to_string()),
"回灌 cookie 应随后续请求发出"
);
assert_eq!(
engine.persistent_cookies().get("x").map(String::as_str),
Some("token=xyz")
);
}
#[tokio::test]
async fn warmup_absorbs_set_cookie_into_jar() {
let json = SOURCE.replacen(
"\"bookInfo\":{}",
"\"enabledCookieJar\":true,\"http\":{\"warmup\":[\"https://x/warm\"]},\"bookInfo\":{}",
1,
);
let src = BookSource::from_json(&json).unwrap();
let last = Arc::new(Mutex::new(None));
let fetcher = Arc::new(CookieEchoFetcher {
set_cookie: "token=warm; Max-Age=3600; Path=/".to_string(),
last_cookie: last.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
engine.warmup().await;
assert_eq!(
engine.persistent_cookies().get("x").map(String::as_str),
Some("token=warm"),
"预热页的 Set-Cookie 应回灌引擎 cookie 库"
);
}
#[tokio::test]
async fn cookie_jar_disabled_does_not_absorb() {
let src = BookSource::from_json(SOURCE).unwrap();
let last = Arc::new(Mutex::new(None));
let fetcher = Arc::new(CookieEchoFetcher {
set_cookie: "token=xyz; Max-Age=3600".to_string(),
last_cookie: last.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
engine.toc("/p1").await.unwrap();
engine.book_info("/p2").await.unwrap();
assert!(
last.lock().unwrap().is_none(),
"未开 cookieJar 不应回灌/再发"
);
assert!(engine.persistent_cookies().is_empty());
}
struct ScriptedFetcher {
routes: Vec<(String, String)>,
calls: Arc<Mutex<Vec<String>>>,
}
#[async_trait]
impl Fetcher for ScriptedFetcher {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
self.calls.lock().unwrap().push(req.url.clone());
for (pat, body) in &self.routes {
if req.url.contains(pat.as_str()) {
return Ok(body.clone());
}
}
Ok(String::new())
}
}
fn scripted(routes: Vec<(&str, &str)>) -> (Arc<ScriptedFetcher>, Arc<Mutex<Vec<String>>>) {
let calls = Arc::new(Mutex::new(Vec::new()));
let f = Arc::new(ScriptedFetcher {
routes: routes
.into_iter()
.map(|(a, b)| (a.to_string(), b.to_string()))
.collect(),
calls: calls.clone(),
});
(f, calls)
}
#[tokio::test]
async fn prelude_captures_token_into_main_request() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{
"prelude":[{"url":{"template":"{{base}}/prepare"},
"capture":[{"name":"token","value":{"via":"raw","clean":[{"trim":true}]},"scope":"chapter"}]}],
"request":{"url":{"template":"{{base}}/search?kw={{key}}&token={{token}}"}},
"list":{"via":"css","select":".item"},
"item":{"name":{"via":"css","select":".t","extract":"text"}}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, calls) = scripted(vec![
("/prepare", "ABC"),
(
"/search",
r#"<div class="item"><span class="t">书名</span></div>"#,
),
]);
let engine = Engine::with_fetcher(src, f);
let items = engine.search("k", 1, 20).await.unwrap().items;
assert_eq!(items.len(), 1);
assert_eq!(items[0].info.name, "书名");
let c = calls.lock().unwrap();
assert!(
c.iter().any(|u| u.contains("/prepare")),
"应先跑前置 prepare: {c:?}"
);
assert!(
c.iter().any(|u| u.contains("token=ABC")),
"主搜索应带捕获的 token: {c:?}"
);
}
#[tokio::test]
async fn skip_if_present_reuses_source_scope_token() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{
"prelude":[{"url":{"template":"{{base}}/prepare"},
"capture":[{"name":"token","value":{"via":"raw","clean":[{"trim":true}]},"scope":"source"}],
"skipIfPresent":["token"]}],
"request":{"url":{"template":"{{base}}/search?token={{token}}"}},
"list":{"via":"css","select":".item"},
"item":{"name":{"via":"css","select":".t","extract":"text"}}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, calls) = scripted(vec![
("/prepare", "TKN"),
(
"/search",
r#"<div class="item"><span class="t">x</span></div>"#,
),
]);
let engine = Engine::with_fetcher(src, f);
engine.search("a", 1, 20).await.unwrap();
engine.search("b", 1, 20).await.unwrap();
let prepares = calls
.lock()
.unwrap()
.iter()
.filter(|u| u.contains("/prepare"))
.count();
assert_eq!(
prepares, 1,
"skipIfPresent 应使 source 级 token 复用,prepare 只跑一次"
);
assert_eq!(
engine.source_vars().get("token").map(String::as_str),
Some("TKN")
);
}
#[tokio::test]
async fn request_vars_visible_to_list_items() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{
"request":{"url":{"template":"{{base}}/s"},
"vars":{"site":{"via":"css","select":".site","extract":"text"}}},
"list":{"via":"css","select":".item"},
"item":{"name":{"template":"{{site}}-书"}}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let html = r#"<span class="site">甲站</span><div class="item">x</div>"#;
let engine = Engine::with_fetcher(src, Arc::new(MockFetcher(html.to_string())));
let items = engine.search("k", 1, 20).await.unwrap().items;
assert_eq!(items.len(), 1);
assert_eq!(
items[0].info.name, "甲站-书",
"item 模板应看到主请求捕获的 site"
);
}
#[tokio::test]
async fn empty_capture_not_written() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{
"prelude":[{"url":{"template":"{{base}}/p"},
"capture":[{"name":"x","value":{"via":"css","select":".nope","extract":"text"},"scope":"source"}]}],
"request":{"url":{"template":"{{base}}/s?x={{x}}"}},
"list":{"via":"css","select":".item"},
"item":{"name":{"via":"css","select":".t","extract":"text"}}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, calls) = scripted(vec![
("/p", "<html></html>"),
("/s", r#"<div class="item"><span class="t">y</span></div>"#),
]);
let engine = Engine::with_fetcher(src, f);
engine.search("k", 1, 20).await.unwrap();
assert!(
!engine.source_vars().contains_key("x"),
"空串捕获不应写作用域层"
);
assert!(
calls.lock().unwrap().iter().any(|u| u.contains("/s?x=")),
"主请求应照常发出(x 为空串)"
);
}
#[tokio::test]
async fn toc_prelude_csrf_visible_to_extraction() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"bookInfo":{},
"toc":{
"prelude":[{"url":{"template":"{{base}}/prepare"},
"capture":[{"name":"csrf","value":{"via":"raw","clean":[{"trim":true}]},"scope":"chapter"}]}],
"list":{"via":"css","select":".ch"},
"name":{"via":"css","select":"a","extract":"text"},
"url":{"concat":[{"literal":"/c?sign="},{"template":"{{csrf}}"},{"literal":"&href="},{"via":"css","select":"a","extract":{"attr":"href"}}]},
"maxPages":1
},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, _calls) = scripted(vec![
("/prepare", "SIG"),
(
"/toc",
r#"<div class="ch"><a href="/n/1.html">第一章</a></div>"#,
),
]);
let engine = Engine::with_fetcher(src, f);
let toc = engine.toc("/toc/1").await.unwrap();
assert_eq!(toc.chapters.len(), 1);
assert_eq!(
toc.chapters[0].url, "/c?sign=SIG&href=/n/1.html",
"目录 url 应拼入前置捕获的 csrf"
);
}
#[tokio::test]
async fn main_request_headers_interpolate_captured_vars() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{
"prelude":[{"url":{"template":"{{base}}/prepare"},
"capture":[{"name":"token","value":{"via":"raw","clean":[{"trim":true}]},"scope":"chapter"}]}],
"request":{"url":{"template":"{{base}}/search"},
"headers":{"Authorization":"Bearer {{token}}"}},
"list":{"via":"css","select":".item"},
"item":{"name":{"via":"css","select":".t","extract":"text"}}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let seen = Arc::new(Mutex::new(None));
struct HeaderProbe {
seen: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl Fetcher for HeaderProbe {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
if req.url.contains("/search") {
*self.seen.lock().unwrap() = req.headers.get("Authorization").cloned();
return Ok(r#"<div class="item"><span class="t">书</span></div>"#.to_string());
}
Ok("ABC".to_string()) }
}
let engine = Engine::with_fetcher(src, Arc::new(HeaderProbe { seen: seen.clone() }));
engine.search("k", 1, 20).await.unwrap();
assert_eq!(
seen.lock().unwrap().clone(),
Some("Bearer ABC".to_string()),
"主请求 header 应插值前置捕获的 token"
);
}
#[tokio::test]
async fn multiple_request_vars_all_captured() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{
"request":{"url":{"template":"{{base}}/s"},
"vars":{
"a":{"via":"css","select":".a","extract":"text"},
"b":{"via":"css","select":".b","extract":"text"}
}},
"list":{"via":"css","select":".item"},
"item":{"name":{"template":"{{a}}-{{b}}"}}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let html = r#"<span class="a">甲</span><span class="b">乙</span><div class="item">x</div>"#;
let engine = Engine::with_fetcher(src, Arc::new(MockFetcher(html.to_string())));
let items = engine.search("k", 1, 20).await.unwrap().items;
assert_eq!(
items[0].info.name, "甲-乙",
"多条 request.vars 应都被捕获且对 item 可见"
);
}
struct RenderProbe {
body: String,
last_render: Arc<Mutex<Option<bool>>>,
}
#[async_trait]
impl Fetcher for RenderProbe {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
*self.last_render.lock().unwrap() = Some(req.render);
Ok(self.body.clone())
}
}
fn explore_source(extra: &str) -> BookSource {
let json = format!(
r#"{{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"explore":{{
"entries":[{{"static":[{{"title":"全部"}}]}}],
"page":{{
"request":{{"url":{{"template":"{{{{base}}}}/lib/page_{{{{page}}}}"}}{extra}}},
"list":{{"via":"css","select":".item"}},
"item":{{"name":{{"via":"css","select":".t","extract":"text"}}}}
}}
}},
"bookInfo":{{}},
"toc":{{"list":{{"via":"css","select":"a"}},"name":{{"via":"css","select":"a"}},"url":{{"via":"css","select":"a","extract":{{"attr":"href"}}}}}},
"content":{{"value":{{"via":"css","select":".c"}}}}
}}"#
);
BookSource::from_json(&json).unwrap()
}
fn explore_entry() -> ExploreEntry {
ExploreEntry {
title: "全部".to_string(),
vars: std::collections::BTreeMap::new(),
}
}
#[tokio::test]
async fn explore_render_uses_render_fetch_path() {
let src = explore_source(r#","render":true,"interceptApi":"book_list/v0""#);
let cat = explore_entry();
let last_render = Arc::new(Mutex::new(None));
let fetcher = Arc::new(RenderProbe {
body: r#"<div class="item"><span class="t">书</span></div>"#.to_string(),
last_render: last_render.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
let items = engine.explore(&cat, 1, 20).await.unwrap().items;
assert_eq!(items.len(), 1);
assert_eq!(items[0].info.name, "书");
assert_eq!(
*last_render.lock().unwrap(),
Some(true),
"explore 开 render 应走渲染取页路径(FetchRequest.render==true)"
);
}
#[tokio::test]
async fn explore_without_render_uses_reqwest_path() {
let src = explore_source("");
let cat = explore_entry();
let last_render = Arc::new(Mutex::new(None));
let fetcher = Arc::new(RenderProbe {
body: r#"<div class="item"><span class="t">书</span></div>"#.to_string(),
last_render: last_render.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
let items = engine.explore(&cat, 1, 20).await.unwrap().items;
assert_eq!(items.len(), 1);
assert_eq!(
*last_render.lock().unwrap(),
Some(false),
"explore 未开 render 应走 reqwest 取页(FetchRequest.render==false)"
);
}
#[tokio::test]
async fn explore_render_failure_degrades_to_err() {
struct FailFetcher;
#[async_trait]
impl Fetcher for FailFetcher {
async fn fetch(&self, _req: FetchRequest) -> std::result::Result<String, FetchError> {
Err(FetchError::Header("渲染拦截超时".into()))
}
}
let src = explore_source(r#","render":true,"interceptApi":"book_list/v0""#);
let cat = explore_entry();
let engine = Engine::with_fetcher(src, Arc::new(FailFetcher));
assert!(
engine.explore(&cat, 1, 20).await.is_err(),
"explore 开 render 取页失败应优雅降级为 Err"
);
}
#[tokio::test]
async fn explore_single_page_fetches_once() {
let src = explore_source("");
let cat = explore_entry();
let (f, calls) = scripted(vec![
(
"page_1",
r#"<div class="item"><span class="t">a</span></div>"#,
),
(
"page_2",
r#"<div class="item"><span class="t">b</span></div>"#,
),
]);
let engine = Engine::with_fetcher(src, f);
let items = engine.explore(&cat, 1, 20).await.unwrap().items;
assert_eq!(items.len(), 1, "explore 单页只取入参 page");
let c = calls.lock().unwrap();
assert_eq!(c.len(), 1, "应只发一次请求: {c:?}");
assert!(c[0].contains("page_1"), "应取起点 page=1: {c:?}");
}
struct DualSourceFetcher {
body: String,
dom: Option<String>,
}
#[async_trait]
impl Fetcher for DualSourceFetcher {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
self.fetch_full(req).await.map(|r| r.body)
}
async fn fetch_full(
&self,
_req: FetchRequest,
) -> std::result::Result<FetchResponse, FetchError> {
Ok(FetchResponse {
body: self.body.clone(),
status: 200,
headers: HashMap::new(),
dom_html: self.dom.clone(),
})
}
}
const PAGINATOR_DOM: &str = r#"<ul class="byte-pagination-list">
<li class="byte-pagination-item byte-pagination-item-icon disabled"></li>
<li class="byte-pagination-item byte-pagination-item-active">1</li>
<li class="byte-pagination-item">2</li>
<li class="byte-pagination-item byte-pagination-item-jumper"></li>
<li class="byte-pagination-item">99</li>
<li class="byte-pagination-item byte-pagination-item-icon"></li>
</ul>"#;
const TP_SELECT: &str =
".byte-pagination-item:not(.byte-pagination-item-icon):not(.byte-pagination-item-jumper)";
#[tokio::test]
async fn total_pages_from_dom_via_css() {
let src = explore_source(&format!(
r#","render":true,"interceptApi":"book_list/v0","readyFor":".byte-pagination","totalPages":{{"via":"css","select":"{TP_SELECT}","index":-1}}"#
));
let cat = explore_entry();
let fetcher = Arc::new(DualSourceFetcher {
body: r#"<div class="item"><span class="t">书</span></div>"#.to_string(),
dom: Some(PAGINATOR_DOM.to_string()),
});
let engine = Engine::with_fetcher(src, fetcher);
let books = engine.explore(&cat, 1, 20).await.unwrap();
assert_eq!(books.items.len(), 1, "list/item 仍对 body 求值");
assert_eq!(
books.total_pages,
Some(99),
"via:css 的 totalPages 应从渲染 DOM 分页器读末页数字"
);
}
#[tokio::test]
async fn total_pages_from_body_when_no_dom() {
let src = explore_source(&format!(
r#","totalPages":{{"via":"css","select":"{TP_SELECT}","index":-1}}"#
));
let cat = explore_entry();
let body = format!(r#"<div class="item"><span class="t">书</span></div>{PAGINATOR_DOM}"#);
let fetcher = Arc::new(DualSourceFetcher { body, dom: None });
let engine = Engine::with_fetcher(src, fetcher);
let books = engine.explore(&cat, 1, 20).await.unwrap();
assert_eq!(
books.total_pages,
Some(99),
"无 DOM 时 totalPages 应对 body(整页 HTML)求值"
);
}
#[tokio::test]
async fn total_pages_none_without_rule() {
let src = explore_source("");
let cat = explore_entry();
let fetcher = Arc::new(DualSourceFetcher {
body: r#"<div class="item"><span class="t">书</span></div>"#.to_string(),
dom: None,
});
let engine = Engine::with_fetcher(src, fetcher);
let books = engine.explore(&cat, 1, 20).await.unwrap();
assert_eq!(books.total_pages, None, "未配 totalPages 应返回 None");
}
fn explore_source_json(extra: &str) -> BookSource {
let json = format!(
r#"{{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"explore":{{
"entries":[{{"static":[{{"title":"全部"}}]}}],
"page":{{
"request":{{"url":{{"template":"{{{{base}}}}/lib/page_{{{{page}}}}"}},"render":true,"interceptApi":"book_list/v0","readyFor":".byte-pagination"{extra}}},
"list":{{"via":"json","select":"$.data.book_list[*]"}},
"item":{{"name":{{"via":"json","select":"$.name"}}}}
}}
}},
"bookInfo":{{}},
"toc":{{"list":{{"via":"css","select":"a"}},"name":{{"via":"css","select":"a"}},"url":{{"via":"css","select":"a","extract":{{"attr":"href"}}}}}},
"content":{{"value":{{"via":"css","select":".c"}}}}
}}"#
);
BookSource::from_json(&json).unwrap()
}
#[tokio::test]
async fn has_more_and_total_pages_coexist_via_routing() {
let src = explore_source_json(&format!(
r#","totalPages":{{"via":"css","select":"{TP_SELECT}","index":-1}},"hasMore":{{"via":"json","select":"$.data.has_more"}}"#
));
let cat = explore_entry();
let fetcher = Arc::new(DualSourceFetcher {
body: r#"{"data":{"book_list":[{"name":"书A"},{"name":"书B"}],"has_more":false}}"#
.to_string(),
dom: Some(PAGINATOR_DOM.to_string()),
});
let engine = Engine::with_fetcher(src, fetcher);
let books = engine.explore(&cat, 1, 20).await.unwrap();
assert_eq!(books.items.len(), 2, "list/item 对 JSON body 求值");
assert_eq!(
books.total_pages,
Some(99),
"totalPages(css)对渲染 DOM 求值"
);
assert_eq!(
books.has_more,
Some(false),
"has_more(json)对 body 求值(非 DOM)——双源路由按 via 区分"
);
}
#[tokio::test]
async fn has_more_true_and_none() {
let src = explore_source_json(r#","hasMore":{"via":"json","select":"$.data.has_more"}"#);
let cat = explore_entry();
let fetcher = Arc::new(DualSourceFetcher {
body: r#"{"data":{"book_list":[{"name":"书"}],"has_more":true}}"#.to_string(),
dom: None,
});
let engine = Engine::with_fetcher(src, fetcher);
assert_eq!(
engine.explore(&cat, 1, 20).await.unwrap().has_more,
Some(true),
"has_more=true → Some(true)"
);
let src2 = explore_source_json("");
let cat2 = explore_entry();
let fetcher2 = Arc::new(DualSourceFetcher {
body: r#"{"data":{"book_list":[{"name":"书"}]}}"#.to_string(),
dom: None,
});
let engine2 = Engine::with_fetcher(src2, fetcher2);
assert_eq!(
engine2.explore(&cat2, 1, 20).await.unwrap().has_more,
None,
"未配 hasMore → None"
);
}
struct PageByProbe {
body: String,
last_page: Arc<Mutex<Option<u32>>>,
last_page_by: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl Fetcher for PageByProbe {
async fn fetch(&self, req: FetchRequest) -> std::result::Result<String, FetchError> {
*self.last_page.lock().unwrap() = Some(req.page);
*self.last_page_by.lock().unwrap() = req.page_by.clone();
Ok(self.body.clone())
}
}
fn search_source(extra: &str) -> BookSource {
let json = format!(
r#"{{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"search":{{
"request":{{"url":{{"template":"{{{{base}}}}/search/{{{{key}}}}"}}{extra}}},
"list":{{"via":"css","select":".item"}},
"item":{{"name":{{"via":"css","select":".t","extract":"text"}}}}
}},
"bookInfo":{{}},
"toc":{{"list":{{"via":"css","select":"a"}},"name":{{"via":"css","select":"a"}},"url":{{"via":"css","select":"a","extract":{{"attr":"href"}}}}}},
"content":{{"value":{{"via":"css","select":".c"}}}}
}}"#
);
BookSource::from_json(&json).unwrap()
}
#[tokio::test]
async fn search_click_pagination_threads_page_and_selector() {
let src = search_source(
r#","render":true,"interceptApi":"search_book/v1","pageBy":{"click":".next"}"#,
);
let last_page = Arc::new(Mutex::new(None));
let last_page_by = Arc::new(Mutex::new(None));
let fetcher = Arc::new(PageByProbe {
body: r#"<div class="item"><span class="t">书</span></div>"#.to_string(),
last_page: last_page.clone(),
last_page_by: last_page_by.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
let items = engine.search("k", 3, 10).await.unwrap().items;
assert_eq!(items.len(), 1);
assert_eq!(
*last_page.lock().unwrap(),
Some(3),
"应把目标页 page=3 透传进请求(供 escalating 点 page-1 次)"
);
assert_eq!(
last_page_by.lock().unwrap().as_deref(),
Some(".next"),
"应把 pageBy.click 选择器透传进请求"
);
}
#[tokio::test]
async fn search_without_page_by_has_no_selector() {
let src = search_source(r#","render":true,"interceptApi":"search_book/v1""#);
let last_page = Arc::new(Mutex::new(None));
let last_page_by = Arc::new(Mutex::new(None));
let fetcher = Arc::new(PageByProbe {
body: r#"<div class="item"><span class="t">书</span></div>"#.to_string(),
last_page: last_page.clone(),
last_page_by: last_page_by.clone(),
});
let engine = Engine::with_fetcher(src, fetcher);
let _ = engine.search("k", 2, 10).await.unwrap();
assert_eq!(*last_page.lock().unwrap(), Some(2));
assert_eq!(
*last_page_by.lock().unwrap(),
None,
"未配 pageBy → 请求不带选择器(现状单拦截)"
);
}
#[test]
fn fanqie_search_config_has_page_by() {
let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../fanqie-web.v2.json");
let Ok(json) = std::fs::read_to_string(path) else {
eprintln!("跳过:未找到 {path}(打包/隔离构建场景)");
return;
};
let src =
BookSource::from_json(&json).expect("fanqie-web.v2.json 应能解析(pageBy 已是已知字段)");
let pb = src
.search
.as_ref()
.expect("应有 search")
.request
.page_by
.as_ref()
.expect("search.request 应配 pageBy(点击翻页)");
assert!(
pb.click.contains("byte-icon-right"),
"pageBy.click 应是番茄 NEXT 选择器: {}",
pb.click
);
let ex = src.explore.as_ref().expect("应有 explore");
assert!(!ex.entries.is_empty(), "explore.entries 应非空");
assert!(
ex.entries
.iter()
.any(|s| matches!(s, crate::source::EntrySource::Static { .. })),
"explore.entries 应含静态入口源(书库·最热/最新)"
);
assert!(ex.page.request.render, "explore.page.request 应开 render");
assert_eq!(
ex.page.request.intercept_api.as_deref(),
Some("book_list/v0"),
"explore.page.request 应拦截 book_list/v0"
);
}
struct CountingFetcher {
body: String,
calls: Arc<Mutex<u32>>,
}
#[async_trait]
impl Fetcher for CountingFetcher {
async fn fetch(&self, _req: FetchRequest) -> std::result::Result<String, FetchError> {
*self.calls.lock().unwrap() += 1;
Ok(self.body.clone())
}
}
fn counting(body: &str) -> (Arc<CountingFetcher>, Arc<Mutex<u32>>) {
let calls = Arc::new(Mutex::new(0));
(
Arc::new(CountingFetcher {
body: body.to_string(),
calls: calls.clone(),
}),
calls,
)
}
#[tokio::test]
async fn render_search_result_is_cached_per_page() {
let src = search_source(
r#","render":true,"interceptApi":"search_book/v1","pageBy":{"click":".next"}"#,
);
let (fetcher, calls) = counting(r#"<div class="item"><span class="t">书</span></div>"#);
let engine = Engine::with_fetcher(src, fetcher);
let a = engine.search("k", 2, 10).await.unwrap();
let b = engine.search("k", 2, 10).await.unwrap(); assert_eq!(
*calls.lock().unwrap(),
1,
"同 (词,页) 第二次应命中缓存,不再取页"
);
assert_eq!(a.items, b.items, "缓存结果应与首次一致");
let _ = engine.search("k", 3, 10).await.unwrap(); assert_eq!(*calls.lock().unwrap(), 2, "不同页应重新取页");
let _ = engine.search("other", 2, 10).await.unwrap(); assert_eq!(*calls.lock().unwrap(), 3, "不同词应重新取页");
}
#[tokio::test]
async fn reqwest_search_is_not_cached() {
let src = search_source(""); let (fetcher, calls) = counting(r#"<div class="item"><span class="t">书</span></div>"#);
let engine = Engine::with_fetcher(src, fetcher);
let _ = engine.search("k", 1, 10).await.unwrap();
let _ = engine.search("k", 1, 10).await.unwrap();
assert_eq!(
*calls.lock().unwrap(),
2,
"非 render 搜索不缓存,两次都应取页"
);
}
#[tokio::test]
async fn explore_static_entry_vars_drive_page_request() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"explore":{
"entries":[ { "static":[
{"title":"全部·最热","vars":{"filter":"all","sort":"hottest"}}
] } ],
"page":{
"request":{
"url":{"template":"{{base}}/library/{{filter}}/page_{{page}}?sort={{sort}}"},
"hasMore":{"via":"json","select":"$.has_more"}
},
"list":{"via":"json","select":"$.books[*]"},
"item":{"name":{"via":"json","select":"$.name"}}
}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, calls) = scripted(vec![(
"/library/all/page_2?sort=hottest",
r#"{"has_more":true,"books":[{"name":"甲"},{"name":"乙"}]}"#,
)]);
let engine = Engine::with_fetcher(src, f);
let entries = engine.explore_entries().await.unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].title, "全部·最热");
let books = engine.explore(&entries[0], 2, 20).await.unwrap();
assert_eq!(books.items.len(), 2);
assert_eq!(books.items[0].info.name, "甲");
assert_eq!(
books.has_more,
Some(true),
"共享 runner 应接好 explore 的 hasMore"
);
let c = calls.lock().unwrap();
assert!(
c.iter()
.any(|u| u.contains("/library/all/page_2?sort=hottest")),
"取页 URL 应由入口变量 filter/sort + page 生成: {c:?}"
);
}
#[tokio::test]
async fn explore_fetch_entries_for_each_merges_and_computes_vars() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"explore":{
"entries":[ { "fetch":{
"forEach":[ {"gender":"0","audience":"女生"}, {"gender":"1","audience":"男生"} ],
"request":{"url":{"template":"{{base}}/cat?gender={{gender}}"}},
"list":{"via":"json","select":"$.data[*]"},
"item":{
"title":{"concat":[{"template":"{{audience}}·"},{"via":"json","select":"$.name"}]},
"vars":{"filter":{"via":"json","select":"$.id"},"sort":{"literal":"hottest"}}
}
} } ],
"page":{
"request":{"url":{"template":"{{base}}/library/{{filter}}/page_{{page}}?sort={{sort}}"}},
"list":{"via":"json","select":"$.books[*]"},
"item":{"name":{"via":"json","select":"$.name"}}
}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, _) = scripted(vec![
("/cat?gender=0", r#"{"data":[{"name":"言情","id":"c1"}]}"#),
(
"/cat?gender=1",
r#"{"data":[{"name":"玄幻","id":"c2"},{"name":"都市","id":"c3"}]}"#,
),
]);
let engine = Engine::with_fetcher(src, f);
let entries = engine.explore_entries().await.unwrap();
assert_eq!(entries.len(), 3);
assert_eq!(entries[0].title, "女生·言情");
assert_eq!(
entries[0].vars.get("filter").map(String::as_str),
Some("c1")
);
assert_eq!(
entries[0].vars.get("sort").map(String::as_str),
Some("hottest")
);
assert_eq!(entries[1].title, "男生·玄幻");
assert_eq!(entries[2].title, "男生·都市");
assert_eq!(
entries[2].vars.get("filter").map(String::as_str),
Some("c3")
);
}
#[tokio::test]
async fn explore_entries_array_merges_static_then_fetch_in_order() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"explore":{
"entries":[
{ "static":[ {"title":"全部·最热","vars":{"filter":"all"}} ] },
{ "fetch":{
"request":{"url":{"template":"{{base}}/cat"}},
"list":{"via":"json","select":"$.data[*]"},
"item":{"title":{"via":"json","select":"$.name"},"vars":{"filter":{"via":"json","select":"$.id"}}}
} }
],
"page":{
"request":{"url":{"template":"{{base}}/library/{{filter}}/page_{{page}}"}},
"list":{"via":"json","select":"$.books[*]"},
"item":{"name":{"via":"json","select":"$.name"}}
}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, _) = scripted(vec![("/cat", r#"{"data":[{"name":"玄幻","id":"c2"}]}"#)]);
let engine = Engine::with_fetcher(src, f);
let entries = engine.explore_entries().await.unwrap();
assert_eq!(entries.len(), 2, "static(1) + fetch(1)");
assert_eq!(entries[0].title, "全部·最热", "静态入口在前");
assert_eq!(entries[1].title, "玄幻", "动态入口在后");
}
#[tokio::test]
async fn explore_entries_partial_success_keeps_static_when_fetch_fails() {
let json = r#"{
"schema":"trnovel-booksource/v2","name":"t","url":"https://x",
"explore":{
"entries":[
{ "static":[ {"title":"静态","vars":{"filter":"all"}} ] },
{ "fetch":{
"request":{"url":{"template":"{{base}}/cat"}},
"list":{"via":"json","select":"$.data[*]"},
"item":{"title":{"via":"json","select":"$.name"}}
} }
],
"page":{
"request":{"url":{"template":"{{base}}/library/{{filter}}/page_{{page}}"}},
"list":{"via":"json","select":"$.books[*]"},
"item":{"name":{"via":"json","select":"$.name"}}
}
},
"bookInfo":{},
"toc":{"list":{"via":"css","select":"a"},"name":{"via":"css","select":"a"},"url":{"via":"css","select":"a","extract":{"attr":"href"}}},
"content":{"value":{"via":"css","select":".c"}}
}"#;
let src = BookSource::from_json(json).unwrap();
let (f, _) = scripted(vec![("/cat", "NOT JSON")]); let engine = Engine::with_fetcher(src, f);
let entries = engine.explore_entries().await.unwrap();
assert_eq!(entries.len(), 1, "fetch 失败但保留静态入口");
assert_eq!(entries[0].title, "静态");
}