dlscan_tools/
lib.rs

1/*
2** EPITECH PROJECT, 2019
3** dlscan
4** File description:
5** lib.rs
6*/
7
8#[macro_use]
9extern crate log;
10extern crate reqwest;
11extern crate scraper;
12
13use reqwest::{
14    blocking::{get, Response},
15    IntoUrl, Result,
16};
17use scraper::{Html, Selector};
18use std::{
19    convert::AsRef, fmt::Display, fs::File, io::copy, path::Path, sync::mpsc::SyncSender,
20    thread::JoinHandle,
21};
22
23pub type Page = (u32, String, String, String, Vec<u8>);
24pub type Tasks = Vec<JoinHandle<()>>;
25pub type Source<'e> = (
26    &'e str,
27    for<'r, 's> fn(&'r [&'s str], SyncSender<Page>) -> Result<()>,
28);
29pub type TitlesList = Vec<(String, Vec<String>)>;
30pub type DbHandler = (JoinHandle<()>, Vec<SyncSender<Page>>);
31
32pub trait Scraping {
33    fn links_from<U, F>(src: U, id: &str, attr: &str, predicate: F) -> Result<Vec<String>>
34    where
35        U: IntoUrl + Display,
36        F: Fn(&str) -> bool,
37    {
38        info!("Reaching links from {} ...", src);
39        Ok(Html::parse_fragment(&get(src)?.text()?)
40            .select(&Selector::parse(id).unwrap())
41            .filter_map(|elem| {
42                if let Some(attr_value) = elem.value().attr(attr) {
43                    if predicate(attr_value) {
44                        return Some(attr_value.to_owned());
45                    }
46                }
47                None
48            })
49            .collect())
50    }
51    fn stack_in_file<P>(mut resp: Response, file: P) -> std::io::Result<()>
52    where
53        P: AsRef<str> + AsRef<Path> + Display,
54    {
55        copy(&mut resp, &mut File::create(&file)?)?;
56        info!("{} - Completed", file);
57        Ok(())
58    }
59}
60
61pub trait ScanScraping: Scraping {
62    fn subpath<T: AsRef<str>>(path: T) -> String;
63    fn gen_list(manga_list: &[&str]) -> Result<Vec<String>>;
64    fn start_dl(manga_list: &[&str], sender: SyncSender<Page>) -> Result<()>;
65    fn dl_chapter(dir: &str, link: &str, sender: &SyncSender<Page>) -> Result<()>;
66}
dlscan_tools/lib.rs

dlscan_tools/
lib.rs
