1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
extern crate dlscan_source;
extern crate json;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate log;
extern crate rayon;
extern crate reqwest;
extern crate rusqlite;
use dlscan_source::*;
use log::LevelFilter;
use rayon::{iter::IntoParallelRefIterator, prelude::*};
use rusqlite::{params, Connection, NO_PARAMS};
use simple_logging::log_to_file;
use std::{
fs::{read_dir, File},
io::{Read, Result},
sync::{
atomic::{AtomicBool, Ordering},
mpsc::{sync_channel, Receiver},
},
thread::{spawn, JoinHandle},
time::Duration,
};
const JSON_PATH: &str = "json";
const LOG_FILE: &str = "scan.log";
const DB_PATH: &str = "scan.db";
const CREATE_MANGATAB: &str = "CREATE TABLE IF NOT EXISTS manga (
name TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE(name,source)
)";
const CREATE_CHAPTERTAB: &str = "CREATE TABLE IF NOT EXISTS page (
id INTEGER,
chapter TEXT NOT NULL,
manga TEXT NOT NULL,
source TEXT NOT NULL,
content BLOB,
UNIQUE(id,chapter,manga,source)
)";
lazy_static! {
static ref CONDVAR: AtomicBool = AtomicBool::new(false);
}
pub struct Launcher;
impl Launcher {
pub fn run() -> Result<()> {
log_to_file(LOG_FILE, LevelFilter::Info)?;
let (co, lists, srcs) = match Self::init_ressources() {
Some(ressources) => ressources,
_ => {
info!("Failed to get ressources - Abort");
return Ok(());
}
};
let (handler, mut snders) = Self::handle_db(srcs, co);
let mut tasks: Tasks = Vec::with_capacity(srcs.len());
srcs.iter().for_each(|src| {
if let Some((_, list)) = lists.clone().into_par_iter().find_any(|x| x.0 == src.0) {
let sdn = snders.remove(0);
tasks.push(spawn(move || {
if let Err(e) =
src.1(&list.par_iter().map(|x| &**x).collect::<Vec<&str>>(), sdn)
{
error!("Download error - cause: {}", e);
}
}));
}
});
Self::join_threads(tasks, handler);
Ok(())
}
fn join_threads(tasks: Vec<JoinHandle<()>>, handler: JoinHandle<()>) {
for task in tasks {
if task.join().is_err() {
warn!("Failed to join source thread");
}
}
CONDVAR.store(true, Ordering::Relaxed);
if handler.join().is_err() {
warn!("Failed to join sql handler");
}
info!("ALL SOURCES COLLECTED");
}
fn extract_titles() -> Result<TitlesList> {
let mut list = Vec::new();
for entry in read_dir(JSON_PATH)?.filter_map(|e| e.ok()) {
let mut buff = String::new();
File::open(&entry.path())?.read_to_string(&mut buff)?;
let name = &*entry.file_name().into_string().unwrap_or_default();
if let Ok(json) = json::parse(&buff) {
list.push((
name.to_lowercase()[0..name.len() - 5].to_owned(),
json.members()
.map(|x| x.dump()[1..x.dump().len() - 1].to_owned())
.collect(),
));
}
}
Ok(list)
}
fn create_db() -> rusqlite::Result<Connection> {
let co = Connection::open(DB_PATH)?;
co.execute(CREATE_MANGATAB, NO_PARAMS)?;
co.execute(CREATE_CHAPTERTAB, NO_PARAMS)?;
Ok(co)
}
fn handle_db(srcs: &[Source], co: Connection) -> DbHandler {
let chans: Vec<(_, Receiver<Page>)> = (0..srcs.len()).map(|_| sync_channel(0)).collect();
let snders = chans.iter().map(|(s, _)| s.clone()).collect();
let rcvers: Vec<_> = chans.into_par_iter().map(|(_, r)| r).collect();
(
spawn(move || {
let mut queue = Vec::new();
while !CONDVAR.load(Ordering::Relaxed) {
for rcv in rcvers.iter() {
if let Ok(page) = rcv.recv_timeout(Duration::from_millis(3)) {
queue.push(page);
}
}
if !queue.is_empty() {
let page = queue.remove(0);
if let Err(e) = co.execute(
"INSERT OR IGNORE INTO manga VALUES (?1, ?2)",
params![page.2, page.3],
) {
error!("Failed to insert values in 'manga' table - cause: {}", e);
}
if let Err(e) = co.execute(
"INSERT OR IGNORE INTO page VALUES (?1, ?2, ?3, ?4, ?5)",
params![page.0, page.1, page.2, page.3, page.4],
) {
error!("Failed to insert values in 'page' table - cause: {}", e);
}
}
}
}),
snders,
)
}
fn init_ressources<'a>() -> Option<(Connection, TitlesList, &'a [Source<'a>])> {
match Self::create_db() {
Ok(co) => match Self::extract_titles() {
Ok(lists) => {
return Some((
co,
lists,
&[
("scantrad", ScanTrad::start_dl),
("lelscan", LelScan::start_dl),
("mangajar", MangaJar::start_dl),
("mangahub", MangaHub::start_dl),
("scanop", ScanOp::start_dl),
],
));
}
Err(e) => error!("Error while extracting lists - cause: {}", e),
},
Err(e) => error!("db error - cause: {}", e),
};
None
}
}