scyros 0.1.0

// Copyright 2025 Andrea Gilot
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Collect metadata of GitHub projects. The input file must be a valid CSV file where one of the columns contains the full
//! names of the projects, and one contains their ids. The program sends requests to the GitHub API to collect metadata about each project. Projects
//! are chosen randomly without replacement. The metadata is saved in a new CSV file. If the program is interrupted, it
//! can be restarted and will continue from where it left off. The program can also optionally use a cache file to save requests.
//! By default, the name of the output file is the same as the input file with the suffix '.metadata.csv'.

use std::collections::HashMap;
use std::collections::HashSet;
use std::io::Write;
use std::iter::FromIterator as _;
use std::path::Path;

use crate::utils::csv::*;
use crate::utils::error::*;
use crate::utils::fs::*;
use crate::utils::github::*;
use crate::utils::github_api::Github;
use crate::utils::json::*;
use crate::utils::logger::Logger;
use clap::ArgAction;
use clap::{Arg, Command};
use indicatif::ProgressBar;
use json::JsonValue;
use polars::frame::DataFrame;
use polars::prelude::*;
use rand::rngs::StdRng;
use rand::seq::SliceRandom as _;
use rand::SeedableRng;

/// Command line arguments parsing.
pub fn cli() -> Command {
    Command::new("metadata")
        .about("Collect the metadata of GitHub projects")
        .long_about(
            "Collect metadata of GitHub projects. The input file must be a valid CSV file where one of the columns (\"name\") contains the full names of the projects, and another one contains their ids.\n\
            The program sends requests to the GitHub API to collect metadata about each project.\n\
            Projects are chosen randomly without replacement. The metadata is saved in a new CSV file.\nIf the program is interrupted, it \
            can be restarted and will continue from where it left off.\nThe program can also optionally use a cache file to save requests.\n\
            By default, the name of the output file is the same as the input file with the suffix '.metadata.csv'.\n"
        )
        .author("Andrea Gilot <andrea.gilot@it.uu.se>")
        .disable_version_flag(true)
        .arg(
            Arg::new("output")
                .short('o')
                .long("output")
                .value_name("OUTPUT_FILE.csv")
                .help("Path to the output csv file to store the metadata. \
                       By default, the name of the output file is the same as the input file with the suffix '.metadata.csv'.")
                .required(false)
        )
        .arg(
            Arg::new("input")
                .short('i')
                .long("input")
                .value_name("INPUT_FILE.csv")
                .help("Path to the input csv file to use. One of the columns must contain the full names of the projects. ")
                .required(true)
        )
        .arg(
            Arg::new("tokens")
                .short('t')
                .long("tokens")
                .value_name("TOKENS_FILE.csv")
                .help("Path to the file containing the GitHub tokens to use. It must be a valid CSV file with one column named 'token' and where every line is a \
                       valid GitHub token (e.g ghp_Ab0C1D2eFg3hIjk4LM56oPqRsTuvWX7yZa8B).")
                .required(true)
        )
        .arg(
            Arg::new("cache")
                .short('c')
                .long("cache")
                .value_name("CACHE.csv")
                .help("Path to the cache file to use. Must have been generated by a previous run of this program.")
                .required(false)
        )
        .arg(
            Arg::new("seed")
                .short('s')
                .long("seed")
                .value_name("SEED")
                .help("Seed used to randomly shuffle the input data.")
                .default_value("2955615809866670875")
                .value_parser(clap::value_parser!(u64)),
        )
        .arg(
            Arg::new("force")
                .short('f')
                .long("force")
                .help("Override the output file if it already exists.")
                .action(ArgAction::SetTrue),
        )
        .arg(
            Arg::new("ids")
                .long("ids")
                .help("Name of the column containing the ids of the projects.")
                .value_name("COLUMN_NAME")
                .default_value("id")
        )
        .arg(
            Arg::new("names")
                .long("names")
                .help("Name of the column containing the full names of the projects.")
                .value_name("COLUMN_NAME")
                .default_value("name")
        )
        .arg(
            Arg::new("sub")
                .long("sub")
                .value_name("NUMBER_OF_PROJECTS")
                .help("Number of projects to sample from the input file. \
                       If not specified, all remaining projects in the input file are used.")
        )
}

/// Collects metadata about GitHub projects.
///
/// The input must be a valid CSV file where the first column is the id of the project and the second column is the full name of the project.
/// Other columns are ignored. Such a file can be obtained by running the random-id-sampling program. Ids are chosen in a random order from the file.
/// The cache file must have been generated by a previous run of this program (possibly with a different input file or seed).
/// The results are saved in a new CSV file with the same name as the input file + ".with_metadata".
/// The output has the following columns:
/// * id: The id of the project.
/// * name: The full name of the project.
/// * language: The main language of the project.
/// * created: The timestamp of the creation of the project.
/// * pushed: The timestamp of the last push to the project.
/// * updated: The timestamp of the last update of the project.
/// * fork: Whether the project is a fork.
/// * disabled: Whether the project is disabled.
/// * archived: Whether the project is archived.
/// * stars: The number of stars of the project.
/// * forks: The number of times the project has been forked.
/// * issues: The number of open issues of the project.
/// * has_issues: Whether the project has issue tracking enabled (different from having 0 issues).
/// * watchers_count: The number of watchers of the project.
/// * subscribers: The number of subscribers of the project.
/// * size: The size of the project in LOC.
/// * license: The name of the license of the project.
///
///
/// # Arguments
///
/// * `input` - The path to the input file.
/// * `tokens` - The path to the file containing the GitHub tokens.
/// * `cache_opt` - The path to the cache file. If not provided, the program will not use a cache.
/// * `seed` - The seed to use for the random number generator.
///
///
/// # Returns
///
/// * Unit if the program finished successfully or an error message if an error occurred.
///
pub fn run(
    input_path: &str,
    output_path: Option<&String>,
    tokens: &str,
    cache_opt: Option<&String>,
    seed: u64,
    force: bool,
    ids: &str,
    names: &str,
    sub: Option<usize>,
    logger: &mut Logger,
) -> Result<(), Error> {
    // Column index of the id in the input and cache files.
    const ID_COL: usize = 0;

    // Check if the token file is valid.
    logger.log_tokens(tokens)?;

    // Load input file
    let input_file: DataFrame = logger.log_completion("Loading input file", || {
        open_csv(
            input_path,
            Some(Schema::from_iter(vec![
                Field::new(ids.into(), DataType::UInt32),
                Field::new(names.into(), DataType::String),
            ])),
            Some(vec![ids, names]),
        )
    })?;

    logger.log_seed(seed)?;

    let mut shuffled_idx: Vec<usize> = (0..input_file.height()).collect();

    // Load the ids from the input file in random order.
    logger.log_completion("Loading project IDs in random order", || {
        let mut rng: StdRng = SeedableRng::seed_from_u64(seed);
        shuffled_idx.shuffle(&mut rng);
        Ok(())
    })?;

    let shuffled_rows = shuffled_idx.into_iter().map(|idx| {
        // Safe unwrap
        let row = input_file.get_row(idx).unwrap().0;

        match (row[0].clone(), row[1].clone()) {
            (AnyValue::UInt32(id), AnyValue::String(name)) => Ok((id, name)),
            _ => Err(idx),
        }
    });

    let n_proj: usize = input_file.height();

    logger.log(&format!("  {} projects found.", n_proj))?;

    // Name of the output file.
    let default_output_path: String = format!("{}.metadata.csv", &input_path);
    let output_file_path: &str = output_path.unwrap_or(&default_output_path);

    // Load the previous results.
    let previous_results: HashSet<u32> = if force {
        HashSet::new()
    } else {
        logger.log_completion("Resuming progress", || {
            Ok(if Path::new(&output_file_path).exists() {
                map_err(
                    map_err(
                        open_csv(
                            input_path,
                            Some(Schema::from_iter(vec![Field::new(
                                ids.into(),
                                DataType::UInt32,
                            )])),
                            Some(vec![ids]),
                        )?
                        .column(ids),
                        "Could not extract the ids from the output file",
                    )?
                    .u32(),
                    "Could not convert the ids to u32",
                )?
                .iter()
                .map(|x| x.unwrap())
                .collect()
            } else {
                HashSet::new()
            })
        })?
    };

    if !previous_results.is_empty() {
        logger.log(&format!(
            "  the metadata of {} projects have already been queried",
            previous_results.len()
        ))?;
    }

    let mut output_file: CSVFile = CSVFile::new(
        output_file_path,
        if force {
            FileMode::Overwrite
        } else {
            FileMode::Append
        },
    )?;

    output_file.write_header(ProjectMetadata::header())?;

    // Load the cache
    let cache: HashMap<u32, String> = logger.log_completion("Loading cache", || {
        Ok(match cache_opt {
            Some(cache_path) => {
                let cache = CSVFile::new(cache_path, FileMode::Read)?;
                cache.indexed_lines(ID_COL)?
            }
            None => HashMap::new(),
        })
    })?;

    logger.log(&format!("  {} projects found in the cache.", cache.len()))?;

    // Number of requests that were saved by using the cache.
    let mut request_from_cache: usize = 0;

    let gh = Github::new(tokens);

    logger.log("Starting to query the GitHub API...")?;

    // Number of projects to sample.
    let mut n: usize = match sub {
        Some(m) => m,
        None => n_proj - previous_results.len(),
    };

    // Create a progress bar
    let progress_bar: ProgressBar = ProgressBar::new(n_proj as u64);

    progress_bar.set_style(
        indicatif::ProgressStyle::default_bar()
            .template("{elapsed} {wide_bar} {percent}% | Requests from cache: {msg}")
            .unwrap(),
    );

    if sub.is_some() {
        progress_bar.set_length(n as u64);
    }

    for row in shuffled_rows {
        if n == 0 {
            break;
        }
        match row {
            Ok((id, full_name)) => {
                // We first check if the project has already been processed. If not, we check the cache.
                // If the project is not in the cache, we make a request to the API.
                if !previous_results.contains(&id) {
                    // Row to write in the output file.
                    let csv_row: String = if cache.contains_key(&id) {
                        // Safe call to unwrap because the key is guaranteed to be in the cache.
                        request_from_cache += 1;
                        cache.get(&id).unwrap().clone()
                    } else {
                        match gh.request(&format!("https://api.github.com/repos/{}", full_name)) {
                            Ok(json) => ProjectMetadata::parse_json(&json, ())?
                                .to_csv((id, full_name.to_string())),
                            Err(e) => ProjectMetadata::default().to_csv((id, e.to_string())),
                        }
                    };

                    map_err(
                        writeln!(&mut output_file, "{}", csv_row),
                        &format!("Could not write to file {}", &output_file_path),
                    )?;

                    progress_bar.inc(1);
                    progress_bar.set_message(request_from_cache.to_string());
                    n -= 1;
                }
            }
            Err(idx) => {
                map_err(
                    row,
                    &format!("Could not parse row {} in the input file", idx),
                )?;
            }
        }
    }
    Ok(())
}

/// Represents the metadata of a GitHub project.
/// The description of the project and the homepage are omitted as they can produce errors in the CSV file.
struct ProjectMetadata {
    /// The main language of the project.
    language: String,
    /// The timestamp of the creation of the project.
    created: i64,
    /// The timestamp of the last push to the project.
    pushed: i64,
    /// The timestamp of the last update of the project.
    updated: i64,
    /// Whether the project is a fork.
    fork: bool,
    /// Whether the project is disabled.
    disabled: bool,
    /// Whether the project is archived.
    archived: bool,
    /// The number of stars of the project.
    stars: u32,
    /// The number of times the project has been forked.
    forks: u32,
    /// The number of open issues of the project.
    issues: u32,
    /// Whether the project has issue tracking enabled (different from having 0 issues).
    has_issues: bool,
    /// The number of watchers of the project.
    watchers_count: u32,
    /// The number of subscribers of the project.
    subscribers: u32,
    /// The size of the project in LOC.
    size: u64,
    /// The name of the license of the project.
    license: String,
}

/// Default implementation for ProjectMetadata.
/// Sets all the dates to 1970-01-01T00:00:00Z, the booleans to false, the numbers to 0 and the strings to empty.
impl Default for ProjectMetadata {
    fn default() -> Self {
        Self {
            language: String::new(),
            created: 0,
            pushed: 0,
            updated: 0,
            fork: false,
            disabled: false,
            archived: false,
            stars: 0,
            forks: 0,
            issues: 0,
            has_issues: false,
            watchers_count: 0,
            subscribers: 0,
            size: 0,
            license: String::new(),
        }
    }
}

impl ToCSV for ProjectMetadata {
    /// Id of the project and error code
    type Key = (u32, String);

    fn header() -> &'static [&'static str] {
        &[
            "id",
            "name",
            "language",
            "created",
            "pushed",
            "updated",
            "fork",
            "disabled",
            "archived",
            "stars",
            "forks",
            "issues",
            "has_issues",
            "watchers_count",
            "subscribers",
            "size",
            "license",
        ]
    }

    fn to_csv(&self, key: Self::Key) -> String {
        format!(
            "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}",
            key.0,
            key.1,
            self.language,
            self.created,
            self.pushed,
            self.updated,
            if self.fork { 1 } else { 0 },
            if self.disabled { 1 } else { 0 },
            if self.archived { 1 } else { 0 },
            self.stars,
            self.forks,
            self.issues,
            if self.has_issues { 1 } else { 0 },
            self.watchers_count,
            self.subscribers,
            self.size,
            self.license,
        )
    }
}

impl FromGitHub for ProjectMetadata {
    type Complement = ();
    fn parse_json(json: &JsonValue, _complement: ()) -> Result<Self, Error> {
        let language = get_field::<String>(json, "language")?;
        let created: i64 = Self::parse_date_time(json, "created_at")?;
        let pushed: i64 = Self::parse_date_time(json, "pushed_at")?;
        let updated: i64 = Self::parse_date_time(json, "updated_at")?;
        let fork = get_field::<bool>(json, "fork")?;
        let disabled = get_field::<bool>(json, "disabled")?;
        let archived = get_field::<bool>(json, "archived")?;
        let stars = get_field::<u32>(json, "stargazers_count")?;
        let forks = get_field::<u32>(json, "forks_count")?;
        let issues = get_field::<u32>(json, "open_issues_count")?;
        let has_issues = get_field::<bool>(json, "has_issues")?;
        let watchers_count = get_field::<u32>(json, "watchers_count")?;
        let subscribers = get_field::<u32>(json, "subscribers_count")?;
        let size = get_field::<u64>(json, "size")?;

        // Parse the license field, defaulting to "unknown" if not present.
        let license = if !json["license"].is_null() {
            clean_string_to_csv(&get_field::<String>(&json["license"], "name")?)
        } else {
            "unknown".to_string()
        };

        // Return a new ProjectMetadata instance with the parsed data.
        Ok(Self {
            language,
            created,
            pushed,
            updated,
            fork,
            disabled,
            archived,
            stars,
            forks,
            issues,
            has_issues,
            watchers_count,
            subscribers,
            size,
            license,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::utils::dataframes::has_column;

    const TEST_DATA: &str = "tests/data/phases/metadata";

    #[test]
    fn test_language_scraper() {
        let input_file: String = format!("{}/repos.csv", TEST_DATA);
        let output_file: String = format!("{}.metadata.csv", input_file);
        assert!(std::path::Path::new(&input_file).exists());

        // Remove the output files if they exist.
        assert!(delete_file(&output_file, true).is_ok());

        let tokens_file: String = "ghtokens.csv".to_string();

        let run_scraper = run(
            &input_file,
            None,
            &tokens_file,
            None,
            0,
            false,
            "id",
            "name",
            None,
            &mut Logger::new(),
        );
        assert!(run_scraper.is_ok());

        let output_df = open_csv(&output_file, None, None);
        assert!(output_df.is_ok());
        let output_df = output_df.unwrap();
        assert!(has_column(&output_df, "name"));
        let sorted_output_df = output_df
            .sort(vec!["name"], SortMultipleOptions::new())
            .unwrap();

        let expected_df = open_csv(&format!("{}.expected", output_file), None, None);
        assert!(expected_df.is_ok());
        let expected_df = expected_df.unwrap();
        assert!(has_column(&expected_df, "name"));
        let sorted_expected_df = expected_df
            .sort(vec!["name"], SortMultipleOptions::new())
            .unwrap();

        assert_eq!(sorted_expected_df, sorted_output_df);

        assert!(delete_file(&output_file, false).is_ok());
    }
}