use std::fs;
use std::sync::Arc;
use anyhow::Context;
use anyhow::Result;
use anyhow::bail;
use chrono::DateTime;
use chrono::TimeZone;
use chrono::Utc;
use curl::easy::Easy2;
use curl::easy::Handler;
use curl::easy::WriteError;
use log::warn;
use sha256::digest;
use crate::Config;
use crate::Inventory;
use crate::License;
use crate::Module;
use crate::Source;
pub fn fetch_documents(config: Arc<Config>, source: Source) -> Result<Vec<Document>> {
let mut documents = vec![];
let documents_dir = config
.site
.files_dir
.join(hashed_subfolder_name(source.clone()));
let full_documents_dir = config
.site
.full_files_dir()
.join(hashed_subfolder_name(source.clone()));
fs::create_dir_all(&full_documents_dir).with_context(|| {
format!(
"unable to create directory `{}`",
full_documents_dir.display()
)
})?;
let mut handle = Easy2::new(Collector(vec![]));
let inventory_url = source.fetch_url().join("inventory.json").unwrap();
handle.get(true).context("unable to set CURLOPT_HTTPGET")?;
handle
.fail_on_error(true)
.context("unable to set CURLOPT_FAILONERROR")?;
handle
.url(inventory_url.as_str())
.context("unable to set CURLOPT_URL")?;
handle
.perform()
.with_context(|| format!("unable to download `{}`", inventory_url))?;
let contents = handle.get_ref();
let inventory = Inventory::load(
String::from_utf8(contents.0.clone())
.context("unable to parse `inventory.json` as UTF-8")?,
config,
)
.context("unable to parse inventory")?;
for (filename, metadata) in &inventory.documents {
let mut handle = Easy2::new(Collector(vec![]));
let document_url = source.fetch_url().join(filename).unwrap();
handle.get(true).context("unable to set CURLOPT_HTTPGET")?;
handle
.fail_on_error(false)
.context("unable to set CURLOPT_FAILONERROR")?;
handle
.fetch_filetime(true)
.context("unable to set CURLOPT_FILETIME")?;
handle
.url(document_url.as_str())
.context("unable to set CURLOPT_URL")?;
handle
.perform()
.with_context(|| format!("unable to download `{}`", document_url))?;
let response_code = handle
.response_code()
.context("unable to get CURLINFO_RESPONSE_CODE")?;
if document_url.scheme() != "file" && response_code != 200 {
warn!(
"unable to download `{}`, got response code {}",
document_url, response_code
);
continue;
}
let filepath = documents_dir.join(filename);
let full_filepath = full_documents_dir.join(filename);
let contents = handle.get_ref();
if let Some(err) = fs::write(&full_filepath, &contents.0).err() {
warn!(
"unable to create file `{}`: {}",
full_filepath.display(),
err
);
continue;
};
documents.push(Document {
filename: match filepath.clone().into_os_string().into_string() {
Ok(s) => s,
Err(_) => bail!("unable to convert `{}` to string", filepath.display()),
},
title: metadata.title.clone().unwrap_or(metadata.module.name()),
authors: if !metadata.authors.is_empty() {
metadata.authors.clone()
} else if !inventory.authors.is_empty() {
inventory.authors.clone()
} else {
vec!["Anon".to_string()]
},
module: metadata.module.clone(),
license: metadata
.license
.clone()
.unwrap_or(inventory.license.clone()),
source: source.clone(),
modification_time: handle
.filetime()
.unwrap_or_default()
.map(|seconds| Utc.timestamp_opt(seconds, 0).unwrap())
.unwrap_or(Utc::now()),
});
}
for entry in fs::read_dir(full_documents_dir).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
let filename = path
.file_name()
.unwrap_or_default()
.to_str()
.unwrap_or_default();
if !inventory.documents.contains_key(filename) {
match fs::remove_file(&path) {
Ok(()) => {}
Err(err) => warn!("unable to remove stale file `{}`: {}", path.display(), err),
};
}
}
Ok(documents)
}
fn hashed_subfolder_name(source: Source) -> String {
let mut hash = digest(source.fetch_url().as_ref());
hash.truncate(20);
hash
}
struct Collector(Vec<u8>);
impl Handler for Collector {
fn write(&mut self, data: &[u8]) -> Result<usize, WriteError> {
self.0.extend_from_slice(data);
Ok(data.len())
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Document {
pub filename: String,
pub title: String,
pub authors: Vec<String>,
pub module: Module,
pub license: License,
pub source: Source,
pub modification_time: DateTime<Utc>,
}
#[cfg(test)]
mod test {
use std::path::PathBuf;
use std::process::Command;
use std::time::Duration;
use assert_fs::prelude::*;
use curl::easy::Easy;
use predicates::prelude::*;
use url::Url;
use super::*;
use crate::config::Provider;
use crate::config::test::example_config;
use crate::config::test::example_licenses;
use crate::inventory::Semester1;
use crate::inventory::Semester2;
#[test]
fn test_fetch_documents() {
compile_example_documents();
let directory = assert_fs::TempDir::new().unwrap();
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = Source {
provider: Provider::File,
owner: String::new(),
repo: String::new(),
custom_display_url: None,
custom_fetch_url: Some(
Url::parse(&format!(
"file://{}/testdata/fetch_documents/",
std::env::current_dir().unwrap().display(),
))
.unwrap(),
),
hide_urls: false,
};
let mut actual_documents = fetch_documents(config.clone(), source.clone()).unwrap();
actual_documents.sort_by_key(|document| document.filename.clone());
let hash = hashed_subfolder_name(source.clone());
assert_eq!(20, hash.len());
let (cc0, cc_by_sa_4_0) = example_licenses();
let expected_documents = vec![
Document {
filename: config
.site
.files_dir
.clone()
.join(&hash)
.join("sem1-analysis-1.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Analysis 1".to_string(),
authors: vec!["Alex".to_string()],
module: Module::Semester1(Semester1::Analysis1),
license: cc0.clone(),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem1-digitaltechnik.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "DT".to_string(),
authors: vec!["Alex".to_string()],
module: Module::Semester1(Semester1::DigitalTechnology),
license: cc0.clone(),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem1-schaltungstheorie.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Schaltungstheorie".to_string(),
authors: vec!["Alex".to_string(), "Max".to_string()],
module: Module::Semester1(Semester1::CircuitTheory),
license: cc_by_sa_4_0.clone(),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem2-analysis-2.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Analysis 2".to_string(),
authors: vec!["Anon".to_string()],
module: Module::Semester2(Semester2::Analysis2),
license: cc0,
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: config
.site
.files_dir
.join(&hash)
.join("sem2-systemtheorie.pdf")
.into_os_string()
.into_string()
.unwrap(),
title: "Systemtheorie".to_string(),
authors: vec!["Max".to_string()],
module: Module::Semester2(Semester2::SystemsTheory),
license: cc_by_sa_4_0,
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
];
pretty_assertions::assert_eq!(expected_documents, actual_documents);
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-analysis-1.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-analysis-1.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-digitaltechnik.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem1-schaltungstheorie.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem2-analysis-2.pdf"),
)
.assert(predicate::path::exists());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("sem2-systemtheorie.pdf"),
)
.assert(predicate::path::exists());
}
#[test]
fn test_fetch_documents_removes_stale_files() {
compile_example_documents();
let directory = assert_fs::TempDir::new().unwrap();
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = Source {
provider: Provider::File,
owner: String::new(),
repo: String::new(),
custom_display_url: None,
custom_fetch_url: Some(
Url::parse(&format!(
"file://{}/testdata/fetch_documents/",
std::env::current_dir().unwrap().display(),
))
.unwrap(),
),
hide_urls: false,
};
let hash = hashed_subfolder_name(source.clone());
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("stale.pdf"),
)
.touch()
.unwrap();
fetch_documents(config.clone(), source.clone()).unwrap();
directory
.child(
PathBuf::from(config.site.files_dir.clone())
.join(&hash)
.join("stale.pdf"),
)
.assert(predicate::path::missing());
}
fn compile_example_documents() {
for entry in fs::read_dir("testdata/fetch_documents/").unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.extension().unwrap() == "typ" {
Command::new("typst")
.args(["compile", path.to_str().unwrap()])
.output()
.expect(&format!("unable to compile `{}`", path.display()));
Command::new("touch")
.args([
"-d",
"@1700000000",
path.with_extension("pdf").to_str().unwrap(),
])
.output()
.expect(&format!(
"unable to set mtime of `{}`",
path.with_extension("pdf").display()
));
}
}
}
#[test]
fn test_fetched_documents_are_not_corrupted() {
let directory = assert_fs::TempDir::new().unwrap();
let config = {
let mut config = example_config();
config.site.output_dir = directory.to_path_buf();
Arc::new(config)
};
let source = Source {
provider: Provider::Codeberg,
owner: "guemax".to_string(),
repo: "kleine-kochbuecher-der-elektrotechnik".to_string(),
custom_display_url: None,
custom_fetch_url: None,
hide_urls: false,
};
let documents = fetch_documents(config.clone(), source.clone()).unwrap();
let running_on_ci = std::env::var("CI").map_or(false, |value| {
if value == "true".to_string() {
true
} else {
false
}
});
if !running_on_ci {
for document in documents {
let full_filepath = config.site.output_dir.join(document.filename);
let output = Command::new("pdfinfo")
.arg(&full_filepath)
.output()
.expect(&format!(
"unable to run `pdfinfo {}`",
full_filepath.display()
));
assert!(
output.status.success(),
"`pdfinfo {}` failed:\nstdout: {}\nstderr: {}",
full_filepath.display(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
}
}
#[test]
fn test_hashed_subfolder_name_collision() {
let source1 = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "cheatsheet-repo-1".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https:/gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
let source2 = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "cheatsheet-repo-2".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https:/gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
let hashed_subfolder_name1 = hashed_subfolder_name(source1);
let hashed_subfolder_name2 = hashed_subfolder_name(source2);
assert!(hashed_subfolder_name1 != hashed_subfolder_name2);
}
#[test]
fn test_codeberg_fetch_url_200_ok_response() {
let source = Source {
provider: Provider::Codeberg,
owner: "guemax".to_string(),
repo: "kleine-kochbuecher-der-elektrotechnik".to_string(),
custom_display_url: None,
custom_fetch_url: None,
hide_urls: false,
};
let mut handle = Easy::new();
handle.get(true).unwrap();
handle.timeout(Duration::from_secs(3)).unwrap();
handle
.url(source.fetch_url().join("inventory.json").unwrap().as_str())
.unwrap();
handle.perform().unwrap();
pretty_assertions::assert_eq!(200, handle.response_code().unwrap());
}
#[test]
fn test_gitea_fetch_url_200_ok_response() {
let source = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "TUM-Formelsammlungen".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https://gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
let mut handle = Easy::new();
handle.get(true).unwrap();
handle.timeout(Duration::from_secs(3)).unwrap();
handle
.url(source.fetch_url().join("inventory.json").unwrap().as_str())
.unwrap();
handle.perform().unwrap();
pretty_assertions::assert_eq!(200, handle.response_code().unwrap());
}
}