1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
#[macro_use]
extern crate log;
extern crate reqwest;
extern crate scraper;
use reqwest::{
blocking::{get, Response},
IntoUrl, Result,
};
use scraper::{Html, Selector};
use std::{
convert::AsRef, fmt::Display, fs::File, io::copy, path::Path, sync::mpsc::SyncSender,
thread::JoinHandle,
};
pub type Page = (u32, String, String, String, Vec<u8>);
pub type Tasks = Vec<JoinHandle<()>>;
pub type Source<'e> = (
&'e str,
for<'r, 's> fn(&'r [&'s str], SyncSender<Page>) -> Result<()>,
);
pub type TitlesList = Vec<(String, Vec<String>)>;
pub type DbHandler = (JoinHandle<()>, Vec<SyncSender<Page>>);
pub trait Scraping {
fn links_from<U, F>(src: U, id: &str, attr: &str, predicate: F) -> Result<Vec<String>>
where
U: IntoUrl + Display,
F: Fn(&str) -> bool,
{
info!("Reaching links from {} ...", src);
Ok(Html::parse_fragment(&get(src)?.text()?)
.select(&Selector::parse(id).unwrap())
.filter_map(|elem| {
if let Some(attr_value) = elem.value().attr(attr) {
if predicate(attr_value) {
return Some(attr_value.to_owned());
}
}
None
})
.collect())
}
fn stack_in_file<P>(mut resp: Response, file: P) -> std::io::Result<()>
where
P: AsRef<str> + AsRef<Path> + Display,
{
copy(&mut resp, &mut File::create(&file)?)?;
info!("{} - Completed", file);
Ok(())
}
}
pub trait ScanScraping: Scraping {
fn subpath<T: AsRef<str>>(path: T) -> String;
fn gen_list(manga_list: &[&str]) -> Result<Vec<String>>;
fn start_dl(manga_list: &[&str], sender: SyncSender<Page>) -> Result<()>;
fn dl_chapter(dir: &str, link: &str, sender: &SyncSender<Page>) -> Result<()>;
}