use std::{collections::{HashMap, HashSet}, fs::{self, File}, io::{self, BufRead, Write}, path::Path, sync::{Arc, Mutex}};
use assist::{Agreement, Institutions};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PDFAgreements {
pub reports: Vec<Report>,
pub all_reports: Vec<AllReport>,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AllReport {
pub label: String,
pub category_code: String,
pub key: i64,
pub owner_institution_id: i64,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Report {
pub label: String,
pub key: i64,
pub owner_institution_id: i64,
}
#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssistCategories {
pub label: String,
pub code: String,
pub report_type: i64,
pub report_category_type: i64,
pub course_transfer_item_type: i64,
pub has_reports: bool,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let institutions = reqwest::get("https://assist.org/api/institutions").await.unwrap().json::<Vec<Institutions>>().await.unwrap();
let ids = institutions.clone().into_iter().map(|x| x.id as usize).collect::<Vec<usize>>();
let mut map = HashMap::new();
for institution in institutions.into_iter() {
map.insert(institution.id, institution.code.trim().to_string());
}
let vec_size = ids.clone().into_iter().max().unwrap() + 1;
let mut relations = Vec::with_capacity(vec_size);
for _ in 0..vec_size {
relations.push(vec![]);
};
let relations = Arc::new(Mutex::new(relations));
let mut futs = FuturesUnordered::new();
let mut outputs = Vec::new();
let threads = 32;
for id in ids {
let relations = Arc::clone(&relations);
let fut = async move {
eprintln!("{}", format!("https://assist.org/api/institutions/{}/agreements", id));
let agreements = reqwest::get(format!("https://assist.org/api/institutions/{}/agreements", id)).await.unwrap().json::<Vec<Agreement>>().await.unwrap();
let mut years = vec![];
for _ in 0..vec_size {
years.push(0);
};
for agree in agreements {
if agree.sending_year_ids.last().is_some() {
if *agree.sending_year_ids.last().unwrap() >= 74 && agree.sending_year_ids.len() >= 2 {
years[agree.institution_parent_id as usize] = agree.sending_year_ids[agree.sending_year_ids.len() - 2];
} else {
years[agree.institution_parent_id as usize] = agree.sending_year_ids.last().unwrap().clone();
}
}
}
relations.lock().unwrap()[id as usize] = years;
};
futs.push(fut);
if futs.len() == threads {
outputs.push(futs.next().await.unwrap());
}
}
while let Some(item) = futs.next().await {
outputs.push(item);
}
let mut checked = HashSet::new();
let mut futs = FuturesUnordered::new();
let mut outputs = Vec::new();
let path = "urls.txt";
let file = match File::open(&path) {
Ok(file) => file,
Err(_) => {File::create_new(&path).unwrap(); File::open(&path).unwrap()},
};
if !Path::new("agreements").exists() {
match fs::create_dir("agreements") {
Ok(_) => println!("Directory created successfully."),
Err(e) => println!("Failed to create directory: {}", e),
}
};
let reader = io::BufReader::new(file);
for line in reader.lines() {
let line = line.unwrap().clone().trim().to_string();
if line.starts_with("http://") || line.starts_with("https://") {
checked.insert(line);
}
}
let relations = Arc::new(relations.lock().unwrap().clone());
let map = Arc::new(map);
for i in 1..=relations.len()-1 {
if !relations[i].is_empty() {
for j in 1..=relations.len()-1 {
if i != j && relations[i][j] != 0 {
if relations[i][j] < 74 {
let relations = Arc::clone(&relations);
let map = Arc::clone(&map);
let fut = async move {
let client = reqwest::ClientBuilder::new().build().unwrap();
let url;
if !client.get(format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=dept", i, j, relations[i][j])).send().await.unwrap().json::<Value>().await.unwrap().get("reports").unwrap().as_array().unwrap().is_empty() {
url = format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=dept", i, j, relations[i][j]);
} else if !client.get(format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=prefix", i, j, relations[i][j])).send().await.unwrap().json::<Value>().await.unwrap().get("reports").unwrap().as_array().unwrap().is_empty() {
url = format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=prefix", i, j, relations[i][j]);
} else if !client.get(format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=major", i, j, relations[i][j])).send().await.unwrap().json::<Value>().await.unwrap().get("reports").unwrap().as_array().unwrap().is_empty() {
url = format!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=major", i, j, relations[i][j]);
} else {
println!("Error: https://assist.org/transfer/results?year={}&institution={}&agreement={}&agreementType=to&view=agreement&viewSendingAgreements=false", relations[i][j], j, i);
return;
}
let agreements = client.get(&url).send().await.unwrap().json::<PDFAgreements>().await.unwrap();
if agreements.all_reports.is_empty() {
for agreement in agreements.reports {
let mut success = false;
while !success {
let file_path = format!("agreements/{}-{}-{}-{}.pdf", relations[i][j], map.get(&(i as u32)).unwrap_or(&i.to_string()), map.get(&(j as u32)).unwrap_or(&j.to_string()), agreement.label.replace("/", "_"));
eprintln!("{}", file_path);
let pdf = client.get(format!("https://assist.org/api/artifacts/{}", agreement.key)).send().await.unwrap().bytes().await.unwrap();
match File::create(&file_path) {
Ok(mut file) => {
if let Err(e) = file.write_all(&pdf) {
println!("Write Error: {}", e);
} else {
success = true;
}
}
Err(e) => {
println!("File Create Error: {} for URL: https://assist.org/api/artifacts/{}", e, agreement.key);
}
}
}
};
} else {
for agreement in agreements.all_reports {
let mut success = false;
while !success {
let file_path = format!("agreements/{}-{}-{}-{}.pdf", relations[i][j], map.get(&(i as u32)).unwrap_or(&i.to_string()), map.get(&(j as u32)).unwrap_or(&j.to_string()), agreement.label.replace("/", "_"));
eprintln!("{}", file_path);
let pdf = client.get(format!("https://assist.org/api/artifacts/{}", agreement.key)).send().await.unwrap().bytes().await.unwrap();
match File::create(&file_path) {
Ok(mut file) => {
if let Err(e) = file.write_all(&pdf) {
println!("Write Error: {}", e);
} else {
success = true;
}
}
Err(e) => {
println!("File Create Error: {} for URL: https://assist.org/api/artifacts/{}", e, agreement.key);
}
}
}
};
};
eprintln!("{}", url);
};
futs.push(fut);
if futs.len() == threads {
outputs.push(futs.next().await.unwrap());
}
}
}
}
}
};
Ok(())
}