use crate::resources::asset::{Asset, AssetKind};
use crate::{Error, file_io, path_io};
use infer::{Infer, MatcherType, Type};
use mime_guess::Mime;
#[cfg(test)]
use mockall::automock;
use std::fmt::{Display, Formatter};
use std::io::Cursor;
use std::path::PathBuf;
use std::str::FromStr;
use std::{
fmt,
fs::{self, File, OpenOptions},
io::{self, Read},
path::Path,
};
use tracing::debug;
#[allow(dead_code)]
pub struct RetrievedContent {
pub reader: Box<dyn Read + Send + Sync + 'static>,
pub mime_type: String,
pub extension: String,
pub size: Option<u64>,
}
impl RetrievedContent {
#[allow(dead_code)]
pub fn new(
reader: Box<dyn Read + Send + Sync + 'static>,
mime_type: String,
extension: String,
size: Option<u64>,
) -> Self {
Self {
reader,
mime_type,
extension,
size,
}
}
}
impl Display for RetrievedContent {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
let size_info = match self.size {
Some(size) => format!("{} bytes", size),
None => "unknown size".to_string(),
};
write!(
f,
"RetrievedContent {{ mime_type: {}, extension: {}, size: {} }}",
self.mime_type, self.extension, size_info
)
}
}
#[derive(Debug)]
pub(crate) struct UpdatedAssetData {
pub(crate) mimetype: Mime,
pub(crate) location_on_disk: PathBuf,
pub(crate) filename: PathBuf,
}
impl Default for UpdatedAssetData {
fn default() -> Self {
UpdatedAssetData {
mimetype: Mime::from_str("plain/txt").unwrap(),
location_on_disk: PathBuf::new(),
filename: PathBuf::new(),
}
}
}
#[cfg_attr(test, automock)]
pub(crate) trait ContentRetriever {
fn download(&self, asset: &Asset) -> Result<UpdatedAssetData, Error>;
fn read(&self, path: &Path, buffer: &mut Vec<u8>) -> Result<(), Error> {
file_io(
file_io(File::open(path), "open-downloaded", path)?.read_to_end(buffer),
"read-downloaded",
path,
)?;
Ok(())
}
fn retrieve(&self, url: &str) -> Result<RetrievedContent, Error>;
}
#[derive(Clone, Debug)]
pub(crate) struct ResourceHandler;
impl ContentRetriever for ResourceHandler {
fn download(&self, asset: &Asset) -> Result<UpdatedAssetData, Error> {
debug!(
"ContentRetriever is going to download asset to dest location = '{:?}'",
asset.location_on_disk
);
if let AssetKind::Remote(url) = &asset.source {
let dest = &asset.location_on_disk;
debug!("Initial asset dest location = '{:?}'", dest);
if dest.is_file() {
debug!("Cache file {:?} to '{}' already exists.", dest, url);
return Ok(UpdatedAssetData {
mimetype: asset.mimetype.clone(),
location_on_disk: asset.location_on_disk.clone(),
filename: asset.filename.clone(),
});
} else {
if let Some(cache_dir) = dest.parent() {
path_io(fs::create_dir_all(cache_dir), cache_dir)?;
}
debug!("Downloading asset by: {}", url);
let mut retrieved_content = self.retrieve(url.as_str())?;
debug!("Retrieved content: \n{}", &retrieved_content);
let mimetype = Mime::from_str(retrieved_content.mime_type.as_str())?;
debug!("Mime from content: \n{:?}", &mimetype);
let mut new_filename = asset.filename.clone();
let mut new_location_on_disk = asset.location_on_disk.clone();
if new_filename.extension().is_none() {
new_filename = PathBuf::from(format!(
"{}.{}",
new_filename.as_os_str().to_str().unwrap(),
retrieved_content.extension
));
new_location_on_disk = PathBuf::from(format!(
"{}.{}",
new_location_on_disk.as_os_str().to_str().unwrap(),
retrieved_content.extension
));
debug!("asset file location: '{:?}'", &new_location_on_disk);
}
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&new_location_on_disk)?;
debug!("File on disk: \n{:?}", &file);
file_io(
io::copy(&mut retrieved_content.reader, &mut file),
"copy-download",
&new_location_on_disk,
)?;
debug!(
"Downloaded asset by '{}' : {:?}",
url, &new_location_on_disk
);
return Ok(UpdatedAssetData {
mimetype,
location_on_disk: new_location_on_disk,
filename: new_filename,
});
}
}
Ok(UpdatedAssetData {
mimetype: asset.mimetype.clone(),
location_on_disk: asset.location_on_disk.clone(),
filename: asset.filename.clone(),
})
}
fn retrieve(&self, url: &str) -> Result<RetrievedContent, Error> {
let res = ureq::get(url).call()?;
match res.status().as_u16() {
200 => {
let mut bytes: Vec<u8> = Vec::with_capacity(1000);
let (parts, body) = res.into_parts();
let _ = body.into_reader().read_to_end(&mut bytes);
let mime_type = parts
.headers
.get("content-type")
.and_then(|val| val.to_str().ok())
.map(|s| s.split(';').next().unwrap_or(s).trim().to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
let infer = Infer::new();
let kind = infer.get(&bytes).unwrap_or_else(|| {
let (matcher_type, mime, extension) = match mime_type.as_str() {
"image/svg+xml" => (MatcherType::Image, "image/svg+xml", "svg"),
"image/png" => (MatcherType::Image, "image/png", "png"),
"image/jpeg" | "image/jpg" => (MatcherType::Image, "image/jpeg", "jpg"),
"image/gif" => (MatcherType::Image, "image/gif", "gif"),
"image/webp" => (MatcherType::Image, "image/webp", "webp"),
"image/x-icon" | "image/vnd.microsoft.icon" => (MatcherType::Image, "image/x-icon", "ico"),
"application/pdf" => (MatcherType::Doc, "application/pdf", "pdf"),
"application/msword" => (MatcherType::Doc, "application/msword", "doc"),
"application/vnd.openxmlformats-officedocument.wordprocessingml.document" => (MatcherType::Doc, "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"),
"application/vnd.ms-excel" => (MatcherType::Doc, "application/vnd.ms-excel", "xls"),
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => (MatcherType::Doc, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"),
"text/html" => (MatcherType::Text, "text/html", "html"),
"text/plain" => (MatcherType::Text, "text/plain", "txt"),
"application/json" => (MatcherType::Text, "application/json", "json"),
"application/xml" | "text/xml" => (MatcherType::Text, "application/xml", "xml"),
"application/zip" => (MatcherType::Archive, "application/zip", "zip"),
"application/x-tar" => (MatcherType::Archive, "application/x-tar", "tar"),
"application/x-rar-compressed" => (MatcherType::Archive, "application/x-rar-compressed", "rar"),
"application/x-7z-compressed" => (MatcherType::Archive, "application/x-7z-compressed", "7z"),
"application/gzip" => (MatcherType::Archive, "application/gzip", "gz"),
"audio/mpeg" => (MatcherType::Audio, "audio/mpeg", "mp3"),
"audio/ogg" => (MatcherType::Audio, "audio/ogg", "ogg"),
"audio/wav" => (MatcherType::Audio, "audio/wav", "wav"),
"video/mp4" => (MatcherType::Video, "video/mp4", "mp4"),
"video/x-matroska" => (MatcherType::Video, "video/x-matroska", "mkv"),
"video/quicktime" => (MatcherType::Video, "video/quicktime", "mov"),
_ => (MatcherType::Custom, "application/octet-stream", "bin"),
};
Type::new(
matcher_type,
mime,
extension,
dummy_check, )
});
let mime_type = kind.mime_type().to_string();
let extension = kind.extension().to_string();
debug!(
"Detected MIME type: {}, Extension: {} for URL: {}",
mime_type, extension, url
);
let content_len = bytes.len() as u64;
let reader: Box<dyn Read + Send + Sync + 'static> = Box::new(Cursor::new(bytes));
Ok(RetrievedContent {
reader,
mime_type,
extension,
size: Some(content_len),
})
}
404 => Err(Error::AssetFileNotFound(format!(
"Missing remote resource: {url}"
))),
_ => unreachable!("Unexpected response status for '{url}'"),
}
}
}
pub fn dummy_check(_buf: &[u8]) -> bool {
true
}
#[cfg(test)]
mod tests {
use crate::errors::Error;
use crate::resources::asset::{Asset, AssetKind};
use mime_guess::Mime;
use std::path::PathBuf;
use tempfile::TempDir;
use tracing::trace;
use url::Url;
use super::{ContentRetriever, ResourceHandler, RetrievedContent, UpdatedAssetData};
#[test]
fn test_download_failed() {
let temp_dir = TempDir::new().unwrap();
let test_dir = temp_dir.path();
let test_url = "https://not_exist.somehost.com/u/274803?v=4";
let asset = Asset {
original_link: test_url.to_string(),
location_on_disk: test_dir.join("downloaded_image"),
filename: PathBuf::from("test_image"),
mimetype: "image/png".parse::<Mime>().unwrap(),
source: AssetKind::Remote(Url::parse(test_url).unwrap()),
};
let handler = ResourceHandler;
let result = handler.download(&asset);
assert!(result.is_err(), "Download should NOT succeed");
}
#[test]
fn test_download_fail_when_resource_not_exist() {
struct TestHandler;
impl ContentRetriever for TestHandler {
fn download(&self, asset: &Asset) -> Result<UpdatedAssetData, Error> {
Err(Error::AssetFileNotFound(format!(
"Missing remote resource: {}",
&asset.original_link.as_str()
)))
}
fn retrieve(&self, url: &str) -> Result<RetrievedContent, Error> {
Err(Error::AssetFileNotFound(format!(
"Missing remote resource: {url}"
)))
}
}
let cr = TestHandler {};
let mut a = temp_remote_asset("https://mdbook-epub.org/not-exist.svg").unwrap();
let r = cr.download(&mut a);
assert!(r.is_err());
assert!(matches!(r.unwrap_err(), Error::AssetFileNotFound(_)));
}
#[test]
#[should_panic(expected = "bad uri: bad url")]
fn test_download_fail_with_unexpected_status() {
struct TestHandler;
impl ContentRetriever for TestHandler {
fn download(&self, _asset: &Asset) -> Result<UpdatedAssetData, Error> {
Err(Error::HttpError(Box::new(ureq::Error::BadUri(
"bad url".to_string(),
))))
}
fn retrieve(&self, _url: &str) -> Result<RetrievedContent, Error> {
panic!("NOT 200 or 404")
}
}
let cr = TestHandler {};
let mut a = temp_remote_asset("https://mdbook-epub.org/bad.svg").unwrap();
let r = cr.download(&mut a);
trace!("{:?}", &r);
panic!("{}", r.unwrap_err().to_string());
}
#[test]
fn test_download_parametrized_avatar_image() {
use std::path::PathBuf;
let temp_dir = TempDir::new().unwrap();
let test_dir = temp_dir.path();
let test_url = "https://avatars.githubusercontent.com/u/274803?v=4";
let asset = Asset {
original_link: test_url.to_string(),
location_on_disk: test_dir.join("downloaded_image"),
filename: PathBuf::from("test_image"),
mimetype: "image/jpg".parse::<Mime>().unwrap(),
source: AssetKind::Remote(Url::parse(test_url).unwrap()),
};
let handler = ResourceHandler;
let result = handler.download(&asset);
assert!(result.is_ok(), "Download should succeed");
let updated_asset = result.unwrap();
assert!(updated_asset.location_on_disk.exists(), "File should exist");
assert!(updated_asset.location_on_disk.is_file(), "Should be a file");
assert_eq!(
updated_asset.location_on_disk.extension().unwrap(),
"jpg",
"File extension should be jpg"
);
let file_size = std::fs::metadata(&updated_asset.location_on_disk)
.unwrap()
.len();
assert!(file_size > 0, "File should not be empty");
assert!(updated_asset.location_on_disk.exists(), "File should exist");
assert!(updated_asset.location_on_disk.is_file(), "Should be a file");
assert_eq!(
updated_asset.location_on_disk.extension().unwrap(),
"jpg",
"File extension should be jpg"
);
let file_size = std::fs::metadata(&updated_asset.location_on_disk)
.unwrap()
.len();
assert!(file_size > 0, "File should not be empty");
assert_eq!(updated_asset.mimetype.to_string(), "image/jpeg");
assert_eq!(
updated_asset.filename.display().to_string(),
"test_image.jpg"
);
}
fn temp_remote_asset(url: &str) -> Result<Asset, Error> {
let tmp_dir = TempDir::new().unwrap();
let dest_dir = tmp_dir.path().join("mdbook-epub");
Asset::from_url(url, url::Url::parse(url).unwrap(), dest_dir.as_path())
}
}