use std::fs;
use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use std::time::SystemTime;
use anyhow::Context;
use anyhow::Result;
use anyhow::bail;
use chrono::DateTime;
use chrono::TimeZone;
use chrono::Utc;
use curl::easy::Easy2;
use curl::easy::Handler;
use curl::easy::WriteError;
use log::warn;
use crate::Config;
use crate::Inventory;
use crate::License;
use crate::Module;
use crate::Source;
static RENDER_TIMEZONE: chrono_tz::Tz = chrono_tz::Tz::Europe__Berlin;
pub fn fetch_documents(config: Arc<Config>, source: Source) -> Result<usize> {
let download_dir = config
.site
.full_files_dir()
.join(source.hashed_subfolder_name());
fs::create_dir_all(&download_dir)
.with_context(|| format!("unable to create directory `{}`", download_dir.display()))?;
fetch_file(source.clone(), "inventory.json".to_string(), &download_dir)?;
let inventory_path = download_dir.join("inventory.json");
let data = fs::read(&inventory_path)
.with_context(|| format!("unable to read from `{}`", inventory_path.display()))?;
let inventory = Inventory::load(
String::from_utf8(data).context("unable to parse `inventory.json` as UTF-8")?,
config.clone(),
)
.context("unable to parse inventory")?;
let mut count = 0;
for filename in inventory.documents.keys() {
if let Err(err) = fetch_file(source.clone(), filename.to_string(), &download_dir) {
warn!("{}", err);
continue;
}
count += 1;
}
for entry in fs::read_dir(download_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
let filename = path
.file_name()
.unwrap_or_default()
.to_str()
.unwrap_or_default();
if !inventory.documents.contains_key(filename) && filename != "inventory.json" {
match fs::remove_file(&path) {
Ok(()) => {}
Err(err) => warn!("unable to remove stale file `{}`: {}", path.display(), err),
};
}
}
Ok(count)
}
fn fetch_file(source: Source, filename: String, download_dir: &Path) -> Result<()> {
let mut handle = Easy2::new(Collector(vec![]));
let document_url = source.fetch_url().join(&filename).unwrap();
handle.get(true).context("unable to set CURLOPT_HTTPGET")?;
handle
.fail_on_error(false)
.context("unable to set CURLOPT_FAILONERROR")?;
handle
.fetch_filetime(true)
.context("unable to set CURLOPT_FILETIME")?;
handle
.url(document_url.as_str())
.context("unable to set CURLOPT_URL")?;
handle
.perform()
.with_context(|| format!("unable to download `{}`", document_url))?;
let response_code = handle
.response_code()
.context("unable to get CURLINFO_RESPONSE_CODE")?;
if document_url.scheme() != "file" && response_code != 200 {
bail!(
"unable to download `{}`, got response code {}",
document_url,
response_code
);
}
let filepath = download_dir.join(filename);
let contents = handle.get_ref();
let mut file = File::create(&filepath)
.with_context(|| format!("unable to create file `{}`", filepath.display()))?;
file.write_all(&contents.0)
.with_context(|| format!("unable to write to file `{}`", filepath.display()))?;
if let Some(time) = SystemTime::UNIX_EPOCH.checked_add(Duration::from_secs(
handle
.filetime()
.unwrap_or_default()
.unwrap_or(
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs()
.try_into()
.unwrap(),
)
.try_into()
.unwrap(),
)) {
file.set_modified(time).with_context(|| {
format!(
"unable to set modification time of file `{}`",
filepath.display()
)
})?;
}
Ok(())
}
struct Collector(Vec<u8>);
impl Handler for Collector {
fn write(&mut self, data: &[u8]) -> Result<usize, WriteError> {
self.0.extend_from_slice(data);
Ok(data.len())
}
}
pub fn load_documents(config: Arc<Config>, source: Source) -> Result<Vec<Document>> {
let mut documents = vec![];
let download_dir = config
.site
.full_files_dir()
.join(source.hashed_subfolder_name());
let inventory_path = download_dir.join("inventory.json");
let data = fs::read(&inventory_path)
.with_context(|| format!("unable to read from `{}`", inventory_path.display()))?;
let inventory = Inventory::load(
String::from_utf8(data).context("unable to parse `inventory.json` as UTF-8")?,
config.clone(),
)
.context("unable to parse inventory")?;
for (filename, metadata) in inventory.documents {
let filepath = config
.site
.files_dir
.join(source.hashed_subfolder_name())
.join(&filename);
let full_filepath = config
.site
.full_files_dir()
.join(source.hashed_subfolder_name())
.join(filename);
documents.push(Document {
filename: match filepath.clone().into_os_string().into_string() {
Ok(s) => s,
Err(_) => bail!("unable to convert `{}` to string", filepath.display()),
},
title: metadata.title.unwrap_or(metadata.module.name()),
authors: if !metadata.authors.is_empty() {
metadata.authors.clone()
} else if !inventory.authors.is_empty() {
inventory.authors.clone()
} else {
vec!["Anon".to_string()]
},
module: metadata.module,
license: metadata.license.unwrap_or(inventory.license.clone()),
source: source.clone(),
modification_time: match modification_time(&full_filepath) {
Ok(modification_time) => modification_time,
Err(err) => {
warn!(
"unable to get modification time of `{}`: {}",
full_filepath.display(),
err
);
Utc::now()
}
},
});
}
Ok(documents)
}
fn modification_time(filepath: &Path) -> Result<DateTime<Utc>> {
Ok(Utc
.timestamp_opt(
File::open(filepath)
.with_context(|| format!("unable to open file `{}`", filepath.display()))?
.metadata()
.with_context(|| {
format!("unable to get metadata of file `{}`", filepath.display())
})?
.modified()
.with_context(|| {
format!(
"unable to get modification time of file `{}`",
filepath.display()
)
})?
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs()
.try_into()
.unwrap(),
0,
)
.unwrap())
}
#[derive(Clone, Debug, PartialEq)]
pub struct Document {
pub filename: String,
pub title: String,
pub authors: Vec<String>,
pub module: Module,
pub license: License,
pub source: Source,
pub modification_time: DateTime<Utc>,
}
impl Document {
pub fn format_timestamp(&self) -> String {
RENDER_TIMEZONE
.from_utc_datetime(&self.modification_time.naive_utc())
.format("%Y-%m-%d %H:%M:%S")
.to_string()
}
}
#[cfg(test)]
mod test {
use std::fs::FileTimes;
use std::path::PathBuf;
use std::process::Command;
use std::time::Duration;
use std::time::UNIX_EPOCH;
use assert_fs::prelude::*;
use predicates::prelude::*;
use url::Url;
use super::*;
use crate::config::Provider;
use crate::config::test::example_config;
use crate::config::test::example_licenses;
use crate::inventory::Semester1;
use crate::inventory::Semester2;
#[test]
fn test_fetch_documents() {
let directory = assert_fs::TempDir::new().unwrap();
let docs_folder = create_example_documents(
directory.to_path_buf(),
UNIX_EPOCH + Duration::from_secs(1_700_000_000),
);
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = example_source(docs_folder);
let actual_count = fetch_documents(config.clone(), source.clone()).unwrap();
let expected_count = 5;
assert_eq!(expected_count, actual_count);
let hash = source.hashed_subfolder_name();
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("inventory.json"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-analysis-1.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-analysis-1.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-digitaltechnik.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-schaltungstheorie.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem2-analysis-2.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem2-systemtheorie.pdf"),
)
.assert(predicate::path::exists());
for entry in fs::read_dir(directory.join(config.site.files_dir.clone()).join(hash)).unwrap()
{
let entry = entry.unwrap();
let path = entry.path();
if path.extension().unwrap_or_default() == "pdf" {
pretty_assertions::assert_eq!(
Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
modification_time(&path).unwrap(),
"{}",
path.display()
);
}
}
}
#[test]
fn test_load_documents() {
let directory = assert_fs::TempDir::new().unwrap();
let docs_folder = create_example_documents(
directory.to_path_buf(),
UNIX_EPOCH + Duration::from_secs(1_700_000_000),
);
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = example_source(docs_folder.clone());
fs::create_dir_all(directory.join("files").join(source.hashed_subfolder_name())).unwrap();
for entry in fs::read_dir(docs_folder.to_string_lossy().to_string()).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
let filename = path
.file_name()
.unwrap_or_default()
.to_str()
.unwrap_or_default();
if path.extension().unwrap_or_default() == "pdf" || filename == "inventory.json" {
let from = path.clone();
let to = directory
.join("files")
.join(source.hashed_subfolder_name())
.join(filename);
let _ = Command::new("cp")
.arg("-p")
.arg(&from)
.arg(&to)
.output()
.expect(&format!(
"unable to copy `{}` to `{}`",
from.display(),
to.display()
));
}
}
let actual_documents = load_documents(config.clone(), source.clone()).unwrap();
let expected_documents: Vec<Document> = example_documents(config.clone(), docs_folder);
pretty_assertions::assert_eq!(expected_documents, actual_documents);
}
#[test]
fn test_fetch_documents_removes_stale_files() {
let directory = assert_fs::TempDir::new().unwrap();
let docs_folder = create_example_documents(
directory.to_path_buf(),
UNIX_EPOCH + Duration::from_secs(1_700_000_000),
);
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = example_source(docs_folder);
let hash = source.hashed_subfolder_name();
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("stale.pdf"),
)
.touch()
.unwrap();
fetch_documents(config.clone(), source.clone()).unwrap();
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("stale.pdf"),
)
.assert(predicate::path::missing());
}
fn create_example_documents(location: PathBuf, time_stamp: SystemTime) -> PathBuf {
for path_str in vec![
"sem1-digitaltechnik.pdf",
"sem1-analysis-1.pdf",
"sem1-schaltungstheorie.pdf",
"sem2-analysis-2.pdf",
"sem2-systemtheorie.pdf",
] {
fs::copy(
"testdata/fetch_and_load_documents/sample.pdf",
location.join(path_str),
)
.unwrap();
let file = File::open(location.join(path_str)).unwrap();
file.set_times(
FileTimes::new()
.set_accessed(time_stamp)
.set_modified(time_stamp),
)
.unwrap();
}
fs::copy(
"testdata/fetch_and_load_documents/inventory.json",
location.join("inventory.json"),
)
.unwrap();
location
}
fn example_source(docs_folder: PathBuf) -> Source {
Source {
provider: Provider::File,
owner: String::new(),
repo: String::new(),
custom_display_url: None,
custom_fetch_url: Some(
Url::parse(&format!("file://{}/", docs_folder.display(),)).unwrap(),
),
hide_urls: false,
}
}
fn example_documents(config: Arc<Config>, docs_folder: PathBuf) -> Vec<Document> {
let source = example_source(docs_folder);
let hash = source.hashed_subfolder_name();
let (cc0, cc_by_sa_4_0) = example_licenses();
let modification_time = Utc.timestamp_opt(1_700_000_000, 0).unwrap();
vec![
Document {
filename: config
.site
.files_dir
.clone()
.join(&hash)
.join("sem1-analysis-1.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Analysis 1".to_string(),
authors: vec!["Alex".to_string()],
module: Module::Semester1(Semester1::Analysis1),
license: cc0.clone(),
source: source.clone(),
modification_time,
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem1-digitaltechnik.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "DT".to_string(),
authors: vec!["Alex".to_string()],
module: Module::Semester1(Semester1::DigitalTechnology),
license: cc0.clone(),
source: source.clone(),
modification_time,
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem1-schaltungstheorie.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Schaltungstheorie".to_string(),
authors: vec!["Alex".to_string(), "Max".to_string()],
module: Module::Semester1(Semester1::CircuitTheory),
license: cc_by_sa_4_0.clone(),
source: source.clone(),
modification_time,
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem2-analysis-2.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Analysis 2".to_string(),
authors: vec!["Anon".to_string()],
module: Module::Semester2(Semester2::Analysis2),
license: cc0,
source: source.clone(),
modification_time,
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem2-systemtheorie.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Systemtheorie".to_string(),
authors: vec!["Max".to_string()],
module: Module::Semester2(Semester2::SystemsTheory),
license: cc_by_sa_4_0,
source: source.clone(),
modification_time,
},
]
}
#[test]
fn test_loaded_documents_are_not_corrupted() {
let directory = assert_fs::TempDir::new().unwrap();
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = Source {
provider: Provider::Codeberg,
owner: "guemax".to_string(),
repo: "kleine-kochbuecher-der-elektrotechnik".to_string(),
custom_display_url: None,
custom_fetch_url: None,
hide_urls: false,
};
let expected_count = fetch_documents(config.clone(), source.clone()).unwrap();
let running_on_ci = std::env::var("CI").map_or(false, |value| {
if value == "true".to_string() {
true
} else {
false
}
});
let mut actual_count = 0;
if !running_on_ci {
for entry in
fs::read_dir(directory.join("files").join(source.hashed_subfolder_name())).unwrap()
{
let entry = entry.unwrap();
let path = entry.path();
if path.extension().unwrap_or_default() == "pdf" {
actual_count += 1;
let output = Command::new("pdfinfo")
.arg(&path)
.output()
.expect(&format!("unable to run `pdfinfo {}`", path.display()));
assert!(
output.status.success(),
"`pdfinfo {}` failed:\nstdout: {}\nstderr: {}",
path.display(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
}
assert_eq!(expected_count, actual_count);
}
}
#[tokio::test]
async fn test_codeberg_load_url_200_ok_response() {
tokio::time::timeout(Duration::from_secs(3), async {
let directory = assert_fs::TempDir::new().unwrap();
let source = Source {
provider: Provider::Codeberg,
owner: "guemax".to_string(),
repo: "kleine-kochbuecher-der-elektrotechnik".to_string(),
custom_display_url: None,
custom_fetch_url: None,
hide_urls: false,
};
fetch_file(source, "inventory.json".to_string(), &directory).unwrap();
directory
.child("inventory.json")
.assert(predicate::path::exists());
})
.await
.unwrap()
}
#[tokio::test]
async fn test_gitea_load_url_200_ok_response() {
tokio::time::timeout(Duration::from_secs(3), async {
let directory = assert_fs::TempDir::new().unwrap();
let source = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "TUM-Formelsammlungen".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https://gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
fetch_file(source, "inventory.json".to_string(), &directory).unwrap();
directory
.child("inventory.json")
.assert(predicate::path::exists());
})
.await
.unwrap()
}
}