use std::{collections::HashSet, fs::{self, File, OpenOptions}, io::{self, BufRead, Write}, path::Path, sync::{Arc, Mutex}};
use assist::{Agreement, AllDepartment, AllMajor, AllPrefix, Articulation, Course, Institution, InstitutionCourseAgreement, Institutions};
use futures::{stream::FuturesUnordered, StreamExt};
use reqwest::ClientBuilder;
use serde_json::{Error, Value};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let institutions = reqwest::get("https://assist.org/api/institutions").await.unwrap().json::<Vec<Institutions>>().await.unwrap();
let ids = institutions.clone().into_iter().map(|x| x.id as usize).collect::<Vec<usize>>();
let vec_size = ids.into_iter().max().unwrap() + 1;
let mut relations = Vec::with_capacity(vec_size);
for _ in 0..vec_size {
relations.push(vec![]);
};
let relations = Arc::new(Mutex::new(relations));
let mut futs = FuturesUnordered::new();
let mut outputs = Vec::new();
let threads = 32;
for i in institutions {
let relations = Arc::clone(&relations);
let fut = async move {
eprintln!("{}", format!("https://assist.org/api/institutions/{}/agreements", i.id));
let agreements = reqwest::get(format!("https://assist.org/api/institutions/{}/agreements", i.id)).await.unwrap().json::<Vec<Agreement>>().await.unwrap();
let mut years = vec![];
for _ in 0..vec_size {
years.push(0);
};
for agree in agreements {
if agree.sending_year_ids.last().is_some() {
years[agree.institution_parent_id as usize] = agree.sending_year_ids.last().unwrap().clone();
}
}
relations.lock().unwrap()[i.id as usize] = years;
};
futs.push(fut);
if futs.len() == threads {
outputs.push(futs.next().await.unwrap());
}
}
while let Some(item) = futs.next().await {
outputs.push(item);
}
let mut urls = Vec::new();
let mut checked = HashSet::new();
let relations = relations.lock().unwrap().clone();
let path = "urls.txt";
let file = match File::open(&path) {
Ok(file) => file,
Err(_) => {File::create_new(&path).unwrap(); File::open(&path).unwrap()},
};
let reader = io::BufReader::new(file);
for line in reader.lines() {
let line = line.unwrap().clone().trim().to_string();
if line.starts_with("http://") || line.starts_with("https://") {
checked.insert(line);
}
}
for i in 1..=relations.len()-1 {
if !relations[i].is_empty() {
for j in 1..=relations.len()-1 {
if i != j && relations[i][j] != 0 {
if relations[i][j] >= 74 {
let url = format!("https://assist.org/api/articulation/Agreements?Key={}/{}/to/{}/AllPrefixes", relations[i][j], j, i);
if !checked.contains(&url) {
urls.push(url);
}
}
}
}
}
};
eprintln!("Urls prepared");
if !Path::new("agreements").exists() {
match fs::create_dir("agreements") {
Ok(_) => println!("Directory created successfully."),
Err(e) => println!("Failed to create directory: {}", e),
}
};
let mut futs = FuturesUnordered::new();
let mut outputs = Vec::new();
let agreements = Arc::new(Mutex::new(Vec::new()));
let threads = 32;
let save_to_file = true;
for url in urls {
let agreements = Arc::clone(&agreements);
let fut = async move {
let client = ClientBuilder::new().build().unwrap();
let agreement;
if client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap().get("isSuccessful").unwrap().to_string().parse::<bool>().unwrap() {
eprintln!("{}", url.clone());
let data = client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap();
let json_str = data.get("result").unwrap().get("articulations").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let articulations: Result<Vec<AllPrefix>, Error> = serde_json::from_str(&json_str[1..=json_str.len()-2]);
if articulations.is_err() {
println!("{}", articulations.unwrap_err());
println!("{}", json_str);
panic!();
}
let articulations = articulations.unwrap();
let receiving_institution = data.get("result").unwrap().get("receivingInstitution").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let sending_institution = data.get("result").unwrap().get("sendingInstitution").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let receiving_institution: Institution = serde_json::from_str(&receiving_institution[1..=receiving_institution.len()-2]).unwrap();
let sending_institution: Institution = serde_json::from_str(&sending_institution[1..=sending_institution.len()-2]).unwrap();
let mut transfers = Vec::new();
for prefix in articulations {
for agreement in prefix.articulations {
transfers.push(agreement);
}
}
agreement = InstitutionCourseAgreement {receiving_institution, sending_institution, courses: transfers};
} else if client.get(url.clone().replace("AllPrefixes", "AllDepartments")).send().await.unwrap().json::<Value>().await.unwrap().get("isSuccessful").unwrap().to_string().parse::<bool>().unwrap() {
let url = url.replace("AllPrefixes", "AllDepartments");
eprintln!("{}", url);
let data = client.get(url.clone().replace("AllPrefixes", "AllMajors")).send().await.unwrap().json::<Value>().await.unwrap();
let json_str = data.get("result").unwrap().get("articulations").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let articulations: Result<Vec<AllDepartment>, Error> = serde_json::from_str(&json_str[1..=json_str.len()-2]);
if articulations.is_err() {
println!("{}", articulations.unwrap_err());
println!("{}", json_str);
panic!();
}
let articulations = articulations.unwrap();
let receiving_institution = data.get("result").unwrap().get("receivingInstitution").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let sending_institution = data.get("result").unwrap().get("sendingInstitution").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let receiving_institution: Institution = serde_json::from_str(&receiving_institution[1..=receiving_institution.len()-2]).unwrap();
let sending_institution: Institution = serde_json::from_str(&sending_institution[1..=sending_institution.len()-2]).unwrap();
let mut transfers = Vec::new();
for dept in articulations {
for agreement in dept.articulations {
transfers.push(agreement);
}
}
agreement = InstitutionCourseAgreement {receiving_institution, sending_institution, courses: transfers };
} else {
if !client.get(url.clone().replace("AllPrefixes", "AllMajors")).send().await.unwrap().json::<Value>().await.unwrap().get("isSuccessful").unwrap().to_string().parse::<bool>().unwrap() {
eprintln!("Error: {}", url);
return;
}
let data = client.get(url.clone().replace("AllPrefixes", "AllMajors")).send().await.unwrap().json::<Value>().await.unwrap();
let json_str = data.get("result").unwrap().get("articulations").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
eprintln!("{}", url.replace("AllPrefixes", "AllMajors"));
let articulations: Result<Vec<AllMajor>, Error> = serde_json::from_str(&json_str[1..=json_str.len()-2]);
if articulations.is_err() {
println!("{}", articulations.unwrap_err());
println!("{}", json_str);
panic!();
}
let articulations = articulations.unwrap();
let receiving_institution = data.get("result").unwrap().get("receivingInstitution").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let sending_institution = data.get("result").unwrap().get("sendingInstitution").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
let receiving_institution: Institution = serde_json::from_str(&receiving_institution[1..=receiving_institution.len()-2]).unwrap();
let sending_institution: Institution = serde_json::from_str(&sending_institution[1..=sending_institution.len()-2]).unwrap();
let mut courses: Vec<Articulation> = articulations.into_iter().map(|x| x.articulation).collect();
courses.retain(|x| x.type_field != "Requirement" );
courses.sort_by(|a, b| {
let binding = Course { id: String::default(), position: 0, course_identifier_parent_id: 0, published_course_identifier_year_term_id: None, denied_on: None, attributes: None, course_title: None, course_number: None, prefix: None, prefix_parent_id: None, prefix_description: None, department_parent_id: None, department: None, begin: None, end: None, min_units: None, max_units: None, pathways: None };
let a_course = a.course.as_ref().unwrap_or(&binding);
let binding = Course { id: String::default(), position: 0, course_identifier_parent_id: 0, published_course_identifier_year_term_id: None, denied_on: None, attributes: None, course_title: None, course_number: None, prefix: None, prefix_parent_id: None, prefix_description: None, department_parent_id: None, department: None, begin: None, end: None, min_units: None, max_units: None, pathways: None };
let b_course = b.course.as_ref().unwrap_or(&binding);
a_course.course_title.cmp(&b_course.course_title)
.then_with(|| a_course.course_number.cmp(&b_course.course_number))
.then_with(|| a_course.prefix.cmp(&b_course.prefix))
});
courses.dedup_by(|a, b| {
let binding = Course { id: String::default(), position: 0, course_identifier_parent_id: 0, published_course_identifier_year_term_id: None, denied_on: None, attributes: None, course_title: None, course_number: None, prefix: None, prefix_parent_id: None, prefix_description: None, department_parent_id: None, department: None, begin: None, end: None, min_units: None, max_units: None, pathways: None };
let a_course = a.course.as_ref().unwrap_or(&binding);
let binding = Course { id: String::default(), position: 0, course_identifier_parent_id: 0, published_course_identifier_year_term_id: None, denied_on: None, attributes: None, course_title: None, course_number: None, prefix: None, prefix_parent_id: None, prefix_description: None, department_parent_id: None, department: None, begin: None, end: None, min_units: None, max_units: None, pathways: None };
let b_course = b.course.as_ref().unwrap_or(&binding);
a_course.course_title == b_course.course_title &&
a_course.course_number == b_course.course_number &&
a_course.prefix == b_course.prefix
});
agreement = InstitutionCourseAgreement { receiving_institution, sending_institution, courses };
}
if save_to_file {
let json_string = serde_json::to_string_pretty(&agreement).unwrap();
let mut file = File::create(format!("./agreements/{}2{}.json", agreement.sending_institution.code.trim().to_lowercase(), agreement.receiving_institution.code.trim().to_lowercase())).unwrap();
file.write_all(json_string.as_bytes()).unwrap();
} else {
agreements.lock().unwrap().push(agreement);
}
let mut file = OpenOptions::new().append(true).open("urls.txt").unwrap();
file.write_all(url.as_bytes()).unwrap();
file.write_all("\n".as_bytes()).unwrap();
};
futs.push(fut);
if futs.len() == threads {
outputs.push(futs.next().await.unwrap());
}
}
while let Some(item) = futs.next().await {
outputs.push(item);
}
eprintln!("Done downloading data");
let agreements = agreements.lock().unwrap().clone();
println!("{}", serde_json::to_string_pretty(&agreements).unwrap());
Ok(())
}