use std::fs;
use std::path::Path;
use anyhow::Context;
use anyhow::Result;
use chrono::DateTime;
use chrono::TimeZone;
use chrono::Utc;
use curl::easy::Easy2;
use curl::easy::Handler;
use curl::easy::WriteError;
use log::warn;
use sha256::digest;
use crate::Inventory;
use crate::Module;
use crate::Source;
pub fn fetch_documents(source: Source, directory: &Path) -> Result<Vec<Document>> {
let mut documents = vec![];
let directory = directory.join(hashed_subfolder_name(source.clone()));
fs::create_dir_all(&directory)
.with_context(|| format!("unable to create directory `{}`", directory.display()))?;
let mut handle = Easy2::new(Collector(vec![]));
let inventory_url = source.fetch_url().join("inventory.json").unwrap();
handle.get(true).context("unable to set CURLOPT_HTTPGET")?;
handle
.fail_on_error(true)
.context("unable to set CURLOPT_FAILONERROR")?;
handle
.url(inventory_url.as_str())
.context("unable to set CURLOPT_URL")?;
handle
.perform()
.with_context(|| format!("unable to download `{}`", inventory_url))?;
let contents = handle.get_ref();
let inventory = Inventory::load(
String::from_utf8(contents.0.clone())
.context("unable to parse `inventory.json` as UTF-8")?,
)
.context("unable to parse inventory")?;
for (filename, metadata) in &inventory.documents {
let mut handle = Easy2::new(Collector(vec![]));
let document_url = source.fetch_url().join(filename).unwrap();
handle.get(true).context("unable to set CURLOPT_HTTPGET")?;
handle
.fail_on_error(false)
.context("unable to set CURLOPT_FAILONERROR")?;
handle
.fetch_filetime(true)
.context("unable to set CURLOPT_FILETIME")?;
handle
.url(document_url.as_str())
.context("unable to set CURLOPT_URL")?;
handle
.perform()
.with_context(|| format!("unable to download `{}`", document_url))?;
let response_code = handle
.response_code()
.context("unable to get CURLINFO_RESPONSE_CODE")?;
if document_url.scheme() != "file" && response_code != 200 {
warn!(
"unable to download `{}`, got response code {}",
document_url, response_code
);
continue;
}
let filepath = directory.join(filename);
let contents = handle.get_ref();
if let Some(err) = fs::write(&filepath, &contents.0).err() {
warn!("unable to create file `{}`: {}", filepath.display(), err);
continue;
};
documents.push(Document {
filename: filepath.to_str().unwrap().to_string(),
title: metadata.title.clone().unwrap_or(metadata.module.name()),
authors: if !metadata.authors.is_empty() {
metadata.authors.clone()
} else if !inventory.authors.is_empty() {
inventory.authors.clone()
} else {
vec!["Anon".to_string()]
},
module: metadata.module.clone(),
source: source.clone(),
modification_time: handle
.filetime()
.unwrap_or_default()
.map(|seconds| Utc.timestamp_opt(seconds, 0).unwrap())
.unwrap_or(Utc::now()),
});
}
for entry in fs::read_dir(directory).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
let filename = path
.file_name()
.unwrap_or_default()
.to_str()
.unwrap_or_default();
if !inventory.documents.contains_key(filename) {
match fs::remove_file(&path) {
Ok(()) => {}
Err(err) => warn!("unable to remove stale file `{}`: {}", path.display(), err),
};
}
}
Ok(documents)
}
fn hashed_subfolder_name(source: Source) -> String {
let mut hash = digest(source.fetch_url().as_ref());
hash.truncate(20);
hash
}
struct Collector(Vec<u8>);
impl Handler for Collector {
fn write(&mut self, data: &[u8]) -> Result<usize, WriteError> {
self.0.extend_from_slice(data);
Ok(data.len())
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Document {
pub filename: String,
pub title: String,
pub authors: Vec<String>,
pub module: Module,
pub source: Source,
pub modification_time: DateTime<Utc>,
}
#[cfg(test)]
mod test {
use std::process::Command;
use std::time::Duration;
use assert_fs::prelude::*;
use curl::easy::Easy;
use predicates::prelude::*;
use url::Url;
use super::*;
use crate::config::Provider;
use crate::inventory::Semester1;
use crate::inventory::Semester2;
#[test]
fn test_fetch_documents() {
compile_example_documents();
let source = Source {
provider: Provider::File,
owner: String::new(),
repo: String::new(),
custom_display_url: None,
custom_fetch_url: Some(
Url::parse(&format!(
"file://{}/testdata/fetch_documents/",
std::env::current_dir().unwrap().display(),
))
.unwrap(),
),
hide_urls: false,
};
let directory = assert_fs::TempDir::new().unwrap();
let mut actual_documents = fetch_documents(source.clone(), directory.path()).unwrap();
actual_documents.sort_by_key(|document| document.filename.clone());
let hash = hashed_subfolder_name(source.clone());
assert_eq!(20, hash.len());
let expected_documents = vec![
Document {
filename: directory
.child(format!("{}/sem1-analysis-1.pdf", hash))
.to_str()
.unwrap()
.to_string(),
title: "Analysis 1".to_string(),
authors: vec!["Alex".to_string()],
module: Module::Semester1(Semester1::Analysis1),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: directory
.child(format!("{}/sem1-digitaltechnik.pdf", hash))
.to_str()
.unwrap()
.to_string(),
title: "DT".to_string(),
authors: vec!["Alex".to_string()],
module: Module::Semester1(Semester1::DigitalTechnology),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: directory
.child(format!("{}/sem1-schaltungstheorie.pdf", hash))
.to_str()
.unwrap()
.to_string(),
title: "Schaltungstheorie".to_string(),
authors: vec!["Alex".to_string(), "Max".to_string()],
module: Module::Semester1(Semester1::CircuitTheory),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: directory
.child(format!("{}/sem2-analysis-2.pdf", hash))
.to_str()
.unwrap()
.to_string(),
title: "Analysis 2".to_string(),
authors: vec!["Anon".to_string()],
module: Module::Semester2(Semester2::Analysis2),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
Document {
filename: directory
.child(format!("{}/sem2-systemtheorie.pdf", hash))
.to_str()
.unwrap()
.to_string(),
title: "Systemtheorie".to_string(),
authors: vec!["Max".to_string()],
module: Module::Semester2(Semester2::SystemsTheory),
source: source.clone(),
modification_time: Utc.timestamp_opt(1_700_000_000, 0).unwrap(),
},
];
pretty_assertions::assert_eq!(expected_documents, actual_documents);
directory
.child(format!("{}/sem1-analysis-1.pdf", hash))
.assert(predicate::path::exists());
directory
.child(format!("{}/sem1-digitaltechnik.pdf", hash))
.assert(predicate::path::exists());
directory
.child(format!("{}/sem1-schaltungstheorie.pdf", hash))
.assert(predicate::path::exists());
directory
.child(format!("{}/sem2-analysis-2.pdf", hash))
.assert(predicate::path::exists());
directory
.child(format!("{}/sem2-systemtheorie.pdf", hash))
.assert(predicate::path::exists());
}
#[test]
fn test_fetch_documents_removes_stale_files() {
compile_example_documents();
let source = Source {
provider: Provider::File,
owner: String::new(),
repo: String::new(),
custom_display_url: None,
custom_fetch_url: Some(
Url::parse(&format!(
"file://{}/testdata/fetch_documents/",
std::env::current_dir().unwrap().display(),
))
.unwrap(),
),
hide_urls: false,
};
let directory = assert_fs::TempDir::new().unwrap();
let hash = hashed_subfolder_name(source.clone());
directory
.child(format!("{}/stale.pdf", hash))
.touch()
.unwrap();
fetch_documents(source.clone(), directory.path()).unwrap();
directory
.child(format!("{}/stale.pdf", hash))
.assert(predicate::path::missing());
}
fn compile_example_documents() {
for entry in fs::read_dir("testdata/fetch_documents/").unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.extension().unwrap() == "typ" {
Command::new("typst")
.args(["compile", path.to_str().unwrap()])
.output()
.expect(&format!("unable to compile `{}`", path.display()));
Command::new("touch")
.args([
"-d",
"@1700000000",
path.with_extension("pdf").to_str().unwrap(),
])
.output()
.expect(&format!(
"unable to set mtime of `{}`",
path.with_extension("pdf").display()
));
}
}
}
#[test]
fn test_fetched_documents_are_not_corrupted() {
let source = Source {
provider: Provider::Codeberg,
owner: "guemax".to_string(),
repo: "kleine-kochbuecher-der-elektrotechnik".to_string(),
custom_display_url: None,
custom_fetch_url: None,
hide_urls: false,
};
let directory = assert_fs::TempDir::new().unwrap();
let documents = fetch_documents(source.clone(), directory.path()).unwrap();
let running_on_ci = std::env::var("CI").map_or(false, |value| {
if value == "true".to_string() {
true
} else {
false
}
});
if !running_on_ci {
for document in documents {
let output = Command::new("pdfinfo")
.arg(document.filename.clone())
.output()
.expect(&format!(
"unable to run `pdfinfo {}`",
document.filename.clone()
));
assert!(
output.status.success(),
"`pdfinfo {}` failed:\nstdout: {}\nstderr: {}",
document.filename,
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
}
}
#[test]
fn test_hashed_subfolder_name_collision() {
let source1 = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "cheatsheet-repo-1".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https:/gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
let source2 = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "cheatsheet-repo-2".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https:/gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
let hashed_subfolder_name1 = hashed_subfolder_name(source1);
let hashed_subfolder_name2 = hashed_subfolder_name(source2);
assert!(hashed_subfolder_name1 != hashed_subfolder_name2);
}
#[test]
fn test_codeberg_fetch_url_200_ok_response() {
let source = Source {
provider: Provider::Codeberg,
owner: "guemax".to_string(),
repo: "kleine-kochbuecher-der-elektrotechnik".to_string(),
custom_display_url: None,
custom_fetch_url: None,
hide_urls: false,
};
let mut handle = Easy::new();
handle.get(true).unwrap();
handle.timeout(Duration::from_secs(3)).unwrap();
handle
.url(source.fetch_url().join("inventory.json").unwrap().as_str())
.unwrap();
handle.perform().unwrap();
pretty_assertions::assert_eq!(200, handle.response_code().unwrap());
}
#[test]
fn test_gitea_fetch_url_200_ok_response() {
let source = Source {
provider: Provider::Gitea,
owner: "alexander".to_string(),
repo: "TUM-Formelsammlungen".to_string(),
custom_display_url: None,
custom_fetch_url: Some(Url::parse("https://gitea.mintcalc.com/").unwrap()),
hide_urls: false,
};
let mut handle = Easy::new();
handle.get(true).unwrap();
handle.timeout(Duration::from_secs(3)).unwrap();
handle
.url(source.fetch_url().join("inventory.json").unwrap().as_str())
.unwrap();
handle.perform().unwrap();
pretty_assertions::assert_eq!(200, handle.response_code().unwrap());
}
}