use std::{collections::{HashMap, HashSet}, path::Path, sync::Arc};
use assist::{Agreement, Institutions};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::{Deserialize, Serialize};
use tokio::{fs::{self, File}, io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, sync::Mutex};
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PDFAgreements {
pub reports: Vec<Report>,
pub all_reports: Vec<AllReport>,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AllReport {
pub label: String,
pub category_code: String,
pub key: i64,
pub owner_institution_id: i64,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Report {
pub label: String,
pub key: i64,
pub owner_institution_id: i64,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssistCategories {
pub label: String,
pub code: String,
pub report_type: i64,
pub report_category_type: i64,
pub course_transfer_item_type: i64,
pub has_reports: bool,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let institutions = reqwest::get("https://assist.org/api/institutions").await.unwrap().json::<Vec<Institutions>>().await.unwrap();
let ids = institutions.clone().into_iter().map(|x| x.id as usize).collect::<Vec<usize>>();
let mut map = HashMap::new();
for institution in institutions.into_iter() {
map.insert(institution.id, institution.code.trim().to_string());
}
let vec_size = ids.clone().into_iter().max().unwrap() + 1;
let mut relations = Vec::with_capacity(vec_size);
for _ in 0..vec_size {
relations.push(vec![]);
};
let relations = Arc::new(Mutex::new(relations));
let mut futs = FuturesUnordered::new();
let mut outputs = Vec::new();
let threads = 32;
let ids = vec![115 as usize];
for id in ids {
let relations = Arc::clone(&relations);
let fut = async move {
eprintln!("{}", format!("https://assist.org/api/institutions/{}/agreements", id));
let agreements = reqwest::get(format!("https://assist.org/api/institutions/{}/agreements", id)).await.unwrap().json::<Vec<Agreement>>().await.unwrap();
let mut years = vec![];
for _ in 0..vec_size {
years.push(vec![]);
};
for agree in agreements {
years[agree.institution_parent_id as usize] = agree.sending_year_ids;
}
relations.lock().await[id as usize] = years;
};
futs.push(fut);
if futs.len() == threads {
outputs.push(futs.next().await.unwrap());
}
}
while let Some(item) = futs.next().await {
outputs.push(item);
}
let mut checked = HashSet::new();
let mut futs = FuturesUnordered::new();
let mut outputs = Vec::new();
let path = "urls.txt";
let file = match File::open(&path).await {
Ok(file) => file,
Err(_) => {File::create_new(&path).await.unwrap(); File::open(&path).await.unwrap()},
};
if !Path::new("agreements").exists() {
match fs::create_dir("agreements").await {
Ok(_) => eprintln!("Directory created successfully."),
Err(e) => eprintln!("Failed to create directory: {}", e),
}
};
let reader = BufReader::new(file);
let mut lines = reader.lines();
while let Some(line) = lines.next_line().await.unwrap() {
let line = line.trim().to_string();
if line.starts_with("http://") || line.starts_with("https://") {
checked.insert(line);
}
}
let relations = Arc::new(relations.lock().await.clone());
let map = Arc::new(map);
let threads = 256;
for i in 1..=relations.len()-1 {
if !relations[i].is_empty() {
for j in 1..=relations.len()-1 {
if i != j && !relations[i][j].is_empty() { for k in 0..=relations[i][j].len()-1 {
if relations[i][j][k] < 74 && relations[i][j][k] > 71 {
let relations = Arc::clone(&relations);
let map = Arc::clone(&map);
let fut = async move {
let client = reqwest::ClientBuilder::new().build().unwrap();
let urls = vec![
format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=dept", i, j, relations[i][j][k]),
format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=prefix", i, j, relations[i][j][k]),
format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=major", i, j, relations[i][j][k]),
format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=breadth", i, j, relations[i][j][k]),
];
for url in urls {
let agreements = client.get(&url).send().await.unwrap().json::<PDFAgreements>().await.unwrap();
for agreement in agreements.reports {
let mut success = false;
while !success {
let directory_path = format!(
"agreements/{}/{}/{}",
relations[i][j][k],
map.get(&(i as u32)).unwrap_or(&i.to_string()),
map.get(&(j as u32)).unwrap_or(&j.to_string())
);
fs::create_dir_all(&directory_path).await.unwrap();
let file_path = format!("{}/{}.pdf", directory_path, agreement.label.replace("/", "_"));
eprintln!("{}", file_path);
let pdf = client.get(format!("https://assist.org/api/artifacts/{}", agreement.key)).send().await.unwrap().bytes().await.unwrap();
match File::create(&file_path).await {
Ok(mut file) => {
if let Err(e) = file.write_all(&pdf).await {
println!("Write Error: {}", e);
} else {
success = true;
}
}
Err(e) => {
println!("File Create Error: {} for URL: https://assist.org/api/artifacts/{}", e, agreement.key);
}
}
}
};
for agreement in agreements.all_reports {
let mut success = false;
while !success {
let directory_path = format!(
"agreements/{}/{}/{}",
relations[i][j][k],
map.get(&(i as u32)).unwrap_or(&i.to_string()),
map.get(&(j as u32)).unwrap_or(&j.to_string())
);
fs::create_dir_all(&directory_path).await.unwrap();
let file_path = format!("{}/{}.pdf", directory_path, agreement.label.replace("/", "_"));
eprintln!("{}", file_path);
let pdf = client.get(format!("https://assist.org/api/artifacts/{}", agreement.key)).send().await.unwrap().bytes().await.unwrap();
match File::create(&file_path).await {
Ok(mut file) => {
if let Err(e) = file.write_all(&pdf).await {
println!("Write Error: {}", e);
} else {
success = true;
}
}
Err(e) => {
println!("File Create Error: {} for URL: https://assist.org/api/artifacts/{}", e, agreement.key);
}
}
}
};
eprintln!("{}", url);
};
};
futs.push(fut);
if futs.len() == threads {
outputs.push(futs.next().await.unwrap());
}
}
}
}
}
}
};
while let Some(item) = futs.next().await {
outputs.push(item);
}
Ok(())
}