use std::{
fmt::Debug,
fs,
io::{BufWriter, Write},
path::{Path, PathBuf},
sync::{Arc, Mutex},
thread,
};
use tempfile::{NamedTempFile, tempdir};
use tracing::error;
use crate::{decompress::DecoderFactory, indicator::IndicatorFactory, utils::Semaphore};
pub mod cas;
pub mod decompress;
pub mod hash;
pub mod indicator;
pub(crate) mod utils;
#[cfg(test)]
mod tests;
pub const CHUNK_SIZE: usize = 1024;
#[derive(Debug, Clone)]
pub struct Decompression {
decoder: fn(Vec<u8>) -> Result<Box<dyn crate::decompress::Decoder>, std::io::Error>,
dst: PathBuf,
exclude: Vec<String>,
}
impl Decompression {
pub fn new<T: DecoderFactory>() -> Self {
Self {
decoder: T::from_bytes,
dst: PathBuf::new(),
exclude: Vec::new(),
}
}
pub fn with_dst<P: AsRef<Path>>(mut self, path: P) -> Self {
self.dst = path.as_ref().to_path_buf();
self
}
pub fn with_exclude<S: ToString>(mut self, str: S) -> Self {
self.exclude.push(str.to_string());
self
}
pub fn with_excludes<S: ToString>(mut self, strs: Vec<S>) -> Self {
self.exclude = strs.iter().map(|e| e.to_string()).collect();
self
}
pub fn extract(self, bytes: Vec<u8>) -> Result<(), String> {
let dir = tempdir().map_err(|e| e.to_string())?;
let mut decoder = (self.decoder)(bytes).map_err(|e| e.to_string())?;
decoder.extract(dir.path().to_path_buf())?;
utils::rcopy(dir, self.dst, self.exclude).map_err(|e| e.to_string())?;
Ok(())
}
pub fn extract_file<P: AsRef<Path>>(self, file: P) -> Result<(), String> {
let bytes = fs::read(file).map_err(|e| e.to_string())?;
self.extract(bytes)
}
}
#[derive(Debug, Clone)]
pub struct File {
pub url: String,
pub path: PathBuf,
pub size: u64,
hash: Option<crate::hash::Hash>,
store: Option<Box<Arc<dyn crate::cas::Store + 'static>>>,
decompression: Option<Decompression>,
}
impl File {
pub fn new(url: &str) -> Self {
Self {
url: url.to_string(),
path: PathBuf::new(),
size: 0,
hash: None,
store: None,
decompression: None,
}
}
pub fn with_path<P: AsRef<Path>>(mut self, path: P) -> Self {
self.path = path.as_ref().to_path_buf();
self
}
pub fn with_size(mut self, size: u64) -> Self {
self.size = size;
self
}
pub fn with_hash(mut self, hash: crate::hash::Hash) -> Self {
self.hash = Some(hash);
self
}
pub fn with_store<T: crate::cas::Store + 'static>(mut self, store: Arc<T>) -> Self {
self.store = Some(Box::new(store));
self
}
pub fn with_decompression(mut self, decompression: Decompression) -> Self {
self.decompression = Some(decompression);
self
}
pub(crate) fn download(
self,
agent: Arc<ureq::Agent>,
indicator: &mut Box<dyn crate::indicator::Indicator + Send>,
) -> Result<(), String> {
if self.path.eq(&PathBuf::new()) {
error!("Please, define the path in the file: {}", self.url);
return Err("Undefined Path".to_string());
}
let request = agent.get(&self.url).call().map_err(|e| e.to_string())?;
if request.status() != 200 {
let error = request.status_text().to_string();
indicator.event(indicator::Event::Error(error.clone()));
return Err(format!("HTTP ERROR: {}", error));
}
let mut current = 0u64;
let path = self.path();
let mut reader = request.into_reader();
if !path.parent().unwrap_or(&path.clone()).exists() {
fs::create_dir_all(&path.parent().unwrap_or(&path.clone())).map_err(|e| e.to_string())?;
}
let mut writer = BufWriter::new(std::fs::File::create(&path).map_err(|e| e.to_string())?);
let size = if self.size == 0 { u64::MAX } else { self.size };
while current < size {
let mut buffer = [0u8; CHUNK_SIZE];
let size = reader.read(&mut buffer).map_err(|e| e.to_string())?;
if size == 0 {
break;
}
current += size as u64;
let buffer = &buffer[0..size];
writer.write(buffer).map_err(|e| e.to_string())?;
indicator.event(indicator::Event::Update(current));
}
writer.flush().map_err(|e| e.to_string())?;
if let Some(hash) = self.hash {
let check = hash.check_file(&path).map_err(|e| e.to_string())?;
if matches!(check, None) {
return Err("Hashes don't matches".to_string());
}
}
if let Some(store) = self.store {
let bytes = fs::read(&path).map_err(|e| e.to_string())?;
fs::remove_file(&path).map_err(|e| e.to_string())?;
store.create(bytes, self.path)?;
}
if let Some(decompression) = self.decompression {
indicator.event(indicator::Event::Stage(String::from("Extracting...")));
decompression.extract_file(&path)?;
}
indicator.event(indicator::Event::End);
Ok(())
}
fn path(&self) -> PathBuf {
if self.store.is_some() {
let path = NamedTempFile::new().unwrap();
return path.path().to_path_buf();
}
self.path.clone()
}
}
pub struct Downloader {
indicator: Box<dyn IndicatorFactory + Send + Sync>,
files: Vec<File>,
max_current_downloads: usize,
agent: Arc<ureq::Agent>,
}
impl Downloader {
pub fn new<T: IndicatorFactory + Sync + Send + 'static>(indicator: T) -> Self {
Self {
indicator: Box::new(indicator),
files: Vec::new(),
max_current_downloads: 5,
agent: Arc::new(ureq::agent()),
}
}
pub fn with_ureq_agent(mut self, agent: ureq::Agent) -> Self {
self.agent = Arc::new(agent);
self
}
pub fn with_max_current_downloads(mut self, max_current_downloads: usize) -> Self {
self.max_current_downloads = max_current_downloads;
self
}
pub fn with_file(mut self, file: File) -> Self {
self.files.push(file);
self
}
pub fn with_files(mut self, files: Vec<File>) -> Self {
self.files = files;
self
}
pub fn with_indicator<T: IndicatorFactory + Send + Sync + 'static>(
mut self,
indicator: T,
) -> Self {
self.indicator = Box::new(indicator);
self
}
pub fn start(self) -> Result<(), String> {
let mut handles = Vec::new();
let semaphore = Arc::new(Semaphore::new(self.max_current_downloads));
let factory = Arc::new(Mutex::new(self.indicator));
let agent = self.agent;
for file in self.files {
let semaphore = semaphore.clone();
let factory = factory.clone();
let agent = agent.clone();
handles.push(thread::spawn(move || {
let factory = factory.clone();
let semaphore = semaphore.clone();
let agent = agent.clone();
semaphore.acquire();
let mut indicator = {
let mut fac = factory.lock().unwrap();
fac.create(
file.path.file_stem().unwrap().to_string_lossy().to_string(),
file.size as usize,
)
};
let err = file.download(agent, &mut indicator);
if let Err(err) = err {
indicator.event(indicator::Event::Error(err));
}
semaphore.release();
}));
}
for handle in handles {
handle.join().unwrap();
}
Ok(())
}
}