assist 0.1.0

use std::{collections::HashSet, fs::{File, OpenOptions}, io::{self, BufRead, Write}, sync::{Arc, Mutex}};

use assist::{Agreement, AllDepartment, AllMajor, Institutions};
use futures::{stream::FuturesUnordered, StreamExt};
use reqwest::ClientBuilder;
use serde_json::Value;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let institutions = reqwest::get("https://assist.org/api/institutions").await.unwrap().json::<Vec<Institutions>>().await.unwrap();
    let ids = institutions.clone().into_iter().map(|x| x.id as usize).collect::<Vec<usize>>();
    let vec_size = ids.into_iter().max().unwrap() + 1;
    let mut relations = Vec::with_capacity(vec_size);
    for _ in 0..vec_size {
        relations.push(vec![]);
    };
    let relations = Arc::new(Mutex::new(relations));
    let mut futs = FuturesUnordered::new();
    let mut outputs = Vec::new();
    let threads = 32;
    for i in institutions {
        let relations = Arc::clone(&relations);
        let fut = async move {
            let agreements = reqwest::get(format!("https://assist.org/api/institutions/{}/agreements", i.id)).await.unwrap().json::<Vec<Agreement>>().await.unwrap();
            let mut years = vec![];
            for _ in 0..vec_size {
                years.push(0);
            };
            for agree in agreements {
                if agree.sending_year_ids.last().is_some() {
                    years[agree.institution_parent_id as usize] = agree.sending_year_ids.last().unwrap().clone();
                }
            }
            relations.lock().unwrap()[i.id as usize] = years;
        };
        futs.push(fut);

        if futs.len() == threads {
            outputs.push(futs.next().await.unwrap());
        }
    }
    while let Some(item) = futs.next().await {
        outputs.push(item);
    }
    /*for i in 0..=relations.len()-1 {
        println!("{}: {:?}", i, relations[i]);
    }
    let mut pairs = Vec::new();
    for i in 0..=vec_size-1 {
        for j in i..=vec_size-1 {
            if i != j && !relations[i].is_empty() && relations[i][j] != 0 {
                pairs.push(((i, j), relations[i][j]));
            }
        }
    }
    println!("{:#?}", pairs);*/
    let mut urls = Vec::new();
    let mut checked = HashSet::new();
    let relations = relations.lock().unwrap().clone();
    let path = "urls.txt";

    let file = File::open(&path).unwrap();
    let reader = io::BufReader::new(file);

    for line in reader.lines() {
        let line = line.unwrap().clone().trim().to_string();
        // Validate and process the URL
        if line.starts_with("http://") || line.starts_with("https://") {
            checked.insert(line);
        }
    }

    for i in 1..=relations.len()-1 {
        if !relations[i].is_empty() {
            for j in 1..=relations.len()-1 {
                if i != j && relations[i][j] != 0 {
                    if relations[i][j] >= 74 {
                        let url = format!("https://assist.org/api/articulation/Agreements?Key={}/{}/to/{}/AllDepartments", relations[i][j], j, i);
                        if !checked.contains(&url) {
                            urls.push(url);
                        }
                    } /*else {
                        println!("https://assist.org/api/agreements?receivingInstitutionId={}&sendingInstitutionId={}&academicYearId={}&categoryCode=major", i, j, relations[i][j]);
                    }*/
                }
            }
        }
    };
    let mut futs = FuturesUnordered::new();
    let mut outputs = Vec::new();
    let threads = 32;
    for url in urls {
        let fut = async move {
            let client = ClientBuilder::new().build().unwrap();
            if !client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap().get("isSuccessful").unwrap().to_string().parse::<bool>().unwrap() {
                let url = url.replace("AllDepartments", "AllMajors");
                eprintln!("{}", url);
                if !client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap().get("isSuccessful").unwrap().to_string().parse::<bool>().unwrap() {
                    println!("Error: {}", url);
                    return;
                }
                let json_str = client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap().get("result").unwrap().get("articulations").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
                eprintln!("{}", url);
                let articulations: Result<Vec<AllMajor>, serde_json::Error> = serde_json::from_str(&json_str[1..=json_str.len()-2]);
                if articulations.is_err() {
                    println!("{}", articulations.unwrap_err());
                    println!("{}", json_str);
                    panic!();
                }
                //println!("{:#?}", articulations);
                if !client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap().get("isSuccessful").unwrap().to_string().parse::<bool>().unwrap() {
                    eprintln!("Error: {}", url);
                }
            } else {
                eprintln!("{}", url);
                let json_str = client.get(url.clone()).send().await.unwrap().json::<Value>().await.unwrap().get("result").unwrap().get("articulations").unwrap().to_string().replace("\\\"", "\"").replace("\\\"", "\"");
                let articulations: Result<Vec<AllDepartment>, serde_json::Error> = serde_json::from_str(&json_str[1..=json_str.len()-2]);
                if articulations.is_err() {
                    println!("{}", articulations.unwrap_err());
                    println!("{}", json_str);
                    panic!();
                }
                //println!("{:#?}", articulations);
            }
            let mut file = OpenOptions::new().append(true).open("urls.txt").unwrap();
            file.write_all(url.as_bytes()).unwrap();
            file.write_all("\n".as_bytes()).unwrap();
        };
        futs.push(fut);

        if futs.len() == threads {
            outputs.push(futs.next().await.unwrap());
        }
    }
    while let Some(item) = futs.next().await {
        outputs.push(item);
    }
    Ok(())
}