acorns 1.2.4

/*
acorns: Generate an AsciiDoc release notes document from tracking tickets.
Copyright (C) 2022  Marek Suchánek  <msuchane@redhat.com>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

use std::collections::HashMap;
use std::convert::From;
use std::default::Default;
use std::fmt;
use std::ops::Neg;
use std::string::ToString;

use askama::Template;
use color_eyre::eyre::{Result, WrapErr};
use counter::Counter;
use once_cell::sync::Lazy;
use regex::Regex;
use serde::Serialize;
use time::{format_description::well_known::Rfc2822, OffsetDateTime};

use crate::extra_fields::DocTextStatus;
use crate::note::content_lines;
use crate::ticket_abstraction::AbstractTicket;
use crate::REGEX_ERROR;

/// These doc types don't belong to any particular target release.
/// Skip the release check for these.
const UNCHECKED_DOC_TYPES: [&str; 3] = [
    "known issue",
    "technology preview",
    "deprecated functionality",
];
/// The maximum allowed title length for a release note.
const MAX_TITLE_LENGTH: usize = 120;

/// A regular expression to extract a version number in the x.y.z format.
static VERSION_XYZ_REGEX: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"(\d+)\.(\d+)\.(\d+)").expect(REGEX_ERROR));
/// A regular expression to extract a version number in the x.y format.
static VERSION_XY_REGEX: Lazy<Regex> =
    Lazy::new(|| Regex::new(r"(\d+)\.(\d+)").expect(REGEX_ERROR));
/// A regular expression to extract a version number in the x (no .y.z) format.
static VERSION_X_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"(\d+)").expect(REGEX_ERROR));

/// An overview of the completeness status across all tickets.
#[derive(Default, Serialize)]
struct OverallProgress {
    all: usize,
    complete: usize,
    complete_pct: f32,
    warnings: usize,
    warnings_pct: f32,
    incomplete: usize,
    incomplete_pct: f32,
}

impl From<&[Checks]> for OverallProgress {
    /// Calculate the global progress statistics for the whole release notes project,
    /// based on the overall status of every ticket.
    fn from(item: &[Checks]) -> Self {
        let all = item.len();
        // TODO: Currently, we calculate the overall checks twice. Once here, and once
        // for the status table. Consolidate to only calculate them once.
        let overall_checks: Vec<Status> = item.iter().map(Checks::overall).collect();
        let complete = overall_checks
            .iter()
            .filter(|status| matches!(status, Status::Ok))
            .count();
        let complete_pct = percentage(complete, all);
        let warnings = overall_checks
            .iter()
            .filter(|status| matches!(status, Status::Warning(_)))
            .count();
        let warnings_pct = percentage(warnings, all);
        let incomplete = overall_checks
            .iter()
            .filter(|status| matches!(status, Status::Error(_)))
            .count();
        let incomplete_pct = percentage(incomplete, all);

        Self {
            all,
            complete,
            complete_pct,
            warnings,
            warnings_pct,
            incomplete,
            incomplete_pct,
        }
    }
}

/// Calculate the percentage of a part in a total amount.
/// Uses `usize` as input because it works with list lengths here.
fn percentage(part: usize, total: usize) -> f32 {
    (part as f32) / (total as f32) * 100.0
}

/// Records all tickets that belong to a writer and stores statistics
/// on the overall completeness of the release notes.
#[derive(Default, Serialize)]
struct WriterStats<'a> {
    name: &'a str,
    total: i32,
    complete: i32,
    warnings: i32,
    incomplete: i32,
}

impl<'a> WriterStats<'a> {
    /// Update these writer statistics with data from a ticket and its release note.
    fn update(&mut self, checks: &Checks) {
        self.total += 1;

        // TODO: This is calculating the overall status once more. Consolidate.
        match checks.overall() {
            Status::Ok => self.complete += 1,
            Status::Warning(_) => self.warnings += 1,
            Status::Error(_) => self.incomplete += 1,
        }
    }

    // TODO: Consolidate with the `percentage` function if possible.
    /// Calculate the percentage of complete release notes assigned to this writer.
    fn percent(&self) -> f32 {
        // If no release notes are assigned to the writer, dividing by 0 would result in NaN.
        // To make the result more readable and useful, report that case as 0% complete.
        if self.total == 0 {
            0.0
        } else {
            (self.complete as f32) / (self.total as f32) * 100.0
        }
    }

    /// Create an instance of `WriterStats` based on a name and initial statistics from a ticket.
    fn new(name: &'a str, checks: &Checks) -> Self {
        let mut stats = WriterStats {
            name,
            // The default numbers are 0.
            ..Default::default()
        };
        // Add to the default 0 values.
        stats.update(checks);

        stats
    }
}

/// Gather statistics on all writers involved in the project and all their release notes.
/// Returns a list of statistics per writer, sorted by the total number of release notes
/// assigned to the writer.
fn calculate_writer_stats<'a>(
    tickets_with_checks: &[(&'a AbstractTicket, &Checks)],
) -> Vec<WriterStats<'a>> {
    let mut writers_map: HashMap<&str, WriterStats> = HashMap::new();

    for (ticket, checks) in tickets_with_checks {
        let name = ticket.docs_contact.as_str();
        writers_map
            .entry(name)
            // If there's already an entry, add new statistics to it based on the current ticket.
            .and_modify(|stats| stats.update(checks))
            // If there isn't an entry yet, initialize it with this ticket's statistics.
            .or_insert(WriterStats::new(name, checks));
    }

    let mut writers: Vec<_> = writers_map.into_values().collect();

    // Sort by the number of assigned release notes in reverse, descending order,
    // so by the negative number of total release notes.
    writers.sort_by_key(|stats| stats.total.neg());

    writers
}

/// Several checks on a ticket, which capture the status of properties
/// relevant to documentation.
#[derive(Default, Serialize)]
struct Checks {
    development: Status,
    doc_type: Status,
    doc_status: Status,
    title_and_text: Status,
    target_release: Status,
}

impl Checks {
    /// Present an overview of all the particular status checks:
    ///
    /// * If any check resulted in an error, return the list of all errors.
    /// * If any check resulted in a warning, return the list of all warnings.
    /// * If there are no errors or warnings, return `Ok`.
    fn overall(&self) -> Status {
        // The text status has a dedicated column in the status table.
        // Its errors might also be long. Because of that, present
        // only a brief error in the overall column instead.
        let short_text_error = Status::Error("Bad text.".into());
        let text_check = match &self.title_and_text {
            Status::Error(_) => &short_text_error,
            other => other,
        };

        // All fields on `Checks`, so that we can iterate over them.
        let items = [
            &self.doc_type,
            text_check,
            &self.doc_status,
            &self.development,
            &self.target_release,
        ];

        // Capture all errors.
        let errors: Vec<&str> = items
            .iter()
            .filter_map(|status| match status {
                Status::Error(e) => Some(e.as_str()),
                _ => None,
            })
            .collect();

        // Capture all warnings.
        let warnings: Vec<&str> = items
            .iter()
            .filter_map(|status| match status {
                Status::Error(e) => Some(e.as_str()),
                _ => None,
            })
            .collect();

        if !errors.is_empty() {
            Status::Error(errors.join(" "))
        } else if !warnings.is_empty() {
            Status::Warning(warnings.join(" "))
        } else {
            Status::Ok
        }
    }
}

/// The status of a particular ticket property. It can be either okay,
/// a non-serious warning with a message, or a serious error with a message.
#[derive(Serialize)]
enum Status {
    Ok,
    Warning(String),
    Error(String),
}

impl Default for Status {
    fn default() -> Self {
        Self::Ok
    }
}

impl Status {
    /// A human-readable status message for this ticket property.
    /// If the status is a warning or an error, provide the message. If it's `Ok`, display `OK`.
    fn message(&self) -> &str {
        match self {
            Self::Ok => "OK",
            Self::Warning(message) | Self::Error(message) => message,
        }
    }

    /// An HTML color associated with a status. It's applied to text in the status table.
    fn color(&self) -> &'static str {
        match self {
            // TODO: Consider tweaking the colors to less obvious, prettier ones.
            Self::Ok => "green",
            Self::Warning(_) => "orange",
            Self::Error(_) => "red",
        }
    }

    // TODO: Consider comparing the doc text with the predefined Bugzilla doc text templates,
    // if Jira also implements them in some way.
    /// Analyze the doc text and check if it conforms to a general release note format.
    fn from_text(text: &str) -> Self {
        let content_lines = content_lines(text);

        match content_lines.len() {
            // If the doc text contains too few paragraphs, return with an error.
            0 => Self::Error("Empty RN.".into()),
            // TODO: If the project configuration auto-generates titles, release notes
            // can normally have just one paragraph. Revisit when the option is available.
            1 => Self::Error("Text in one paragraph.".into()),
            _ => {
                // If the doc text contains at least two paragraphs, it can be a release note.
                // In that case, proceed with the analysis.
                // It's now safe to index directly into the list, because it contains at least 2 items.
                // Use this to analyze the release note title in detail.
                let first_content_line = content_lines[0];
                Self::from_title(first_content_line)
            }
        }
    }

    /// Check that the first line in a release note is a title
    /// in the AsciiDoc label format, and that it matches other title requirements.
    fn from_title(text: &str) -> Self {
        let old_title_regex = Regex::new(r"^ *\.(\S+.*)").expect(REGEX_ERROR);
        let new_title_regex = Regex::new(r"^ *(\S+.*)::$").expect(REGEX_ERROR);

        let (title, is_legacy) = if let Some(caps) = new_title_regex.captures(text) {
            (caps.get(1).map(|m| m.as_str()).unwrap_or(""), false)
        } else if let Some(caps) = old_title_regex.captures(text) {
            (caps.get(1).map(|m| m.as_str()).unwrap_or(""), true)
        } else {
            return Self::Error("Missing title.".into());
        };

        let length = title.chars().count();

        if text.starts_with(' ') {
            Self::Error("Title starts with a space.".into())
        } else if length > MAX_TITLE_LENGTH {
            Self::Warning(format!("Long title: {length} characters."))
        } else if is_legacy {
            Self::Warning("OK (legacy format)".into())
        } else {
            Self::Ok
        }
    }

    /// Report when the bug is in early stages of development.
    fn from_devel_status(status: &str) -> Self {
        match status.to_lowercase().as_str() {
            "to do" | "new" | "assigned" | "modified" => Self::Warning("Early development.".into()),
            _ => Self::Ok,
        }
    }

    /// Report if the doc type is set to a non-release note type.
    fn from_doc_type(doc_type: &str) -> Self {
        match doc_type {
            "If docs needed, set a value" => Self::Error("Bad doc type.".into()),
            _ => Self::Ok,
        }
    }

    /// Report if the ticket's target release doesn't match the the global target release.
    fn from_target_release(
        ticket_releases: &[String],
        likely_release: Option<Version>,
        doc_type: &str,
    ) -> Self {
        if let Some(likely_release) = likely_release {
            // This is a replacement to the `contains` method that converts the `String` list to `&str`,
            // and thus enables us to compare the two strings without allocating every time.
            if ticket_releases
                .iter()
                .any(|r| Version::from(r) == likely_release)
                || UNCHECKED_DOC_TYPES.contains(&doc_type.to_lowercase().as_str())
            {
                Self::Ok
            } else {
                Self::Warning("Check target release.".into())
            }
        } else {
            Self::Ok
        }
    }
}

impl From<DocTextStatus> for Status {
    fn from(item: DocTextStatus) -> Self {
        match item {
            DocTextStatus::Approved => Self::Ok,
            DocTextStatus::InProgress => Self::Error("RN not approved.".into()),
            DocTextStatus::NoDocumentation => Self::Error("RN not needed.".into()),
        }
    }
}

impl AbstractTicket {
    /// Analyze the release note status of the ticket. Record the analysis as `Checks`.
    fn checks(&self, release: Option<Version>) -> Checks {
        Checks {
            development: Status::from_devel_status(&self.status),
            title_and_text: Status::from_text(&self.doc_text),
            doc_type: Status::from_doc_type(&self.doc_type),
            doc_status: Status::from(self.doc_text_status),
            target_release: Status::from_target_release(
                &self.target_releases,
                release,
                &self.doc_type,
            ),
        }
    }

    /// Extract the account name before `@` from the docs contact email address.
    fn docs_contact_short(&self) -> &str {
        email_prefix(self.docs_contact.as_str())
    }

    /// Extract the account name before `@` from the assignee email address.
    fn assignee_short(&self) -> &str {
        if let Some(assignee) = &self.assignee {
            email_prefix(assignee)
        } else {
            "No assignee"
        }
    }

    /// Display the list of flags or labels for this ticket, depending on which it contains.
    fn flags_or_labels(&self) -> String {
        // TODO: Maybe combine flags and labels together as one list?
        if let Some(flags) = &self.flags {
            flags.join(", ")
        } else if let Some(labels) = &self.labels {
            labels.join(", ")
        } else {
            "No flags or labels".to_string()
        }
    }

    /// Display the list of target releases, or a placeholder if there are none.
    fn display_target_releases(&self) -> String {
        if self.target_releases.is_empty() {
            "No releases".to_string()
        } else {
            self.target_releases.join(", ")
        }
    }

    /// Display the list of subsystems, or a placeholder if there are none.
    fn display_subsystems(&self) -> String {
        match &self.subsystems {
            Ok(subsystems) => {
                if subsystems.is_empty() {
                    "No subsystems".to_string()
                } else {
                    subsystems.join(", ")
                }
            }
            // If getting the subsystems field resulted in an error, it's not
            // a fatal issue in the status table. Just report it and proceed.
            Err(_) => "Invalid subsystems".to_string(),
        }
    }

    /// Display the list of components, or a placeholder if there are none.
    fn display_components(&self) -> String {
        if self.components.is_empty() {
            "No components".to_string()
        } else {
            self.components.join(", ")
        }
    }

    /// Display the status, and if closed, also the resolution.
    fn display_status(&self) -> String {
        // For closed tickets, attach the resolution to the status.
        if self.status.to_lowercase() == "closed" {
            let resolution = match &self.resolution {
                Some(resolution) => resolution.as_str(),
                None => "no resolution",
            };
            format!("{}: {}", self.status, resolution)
        // For open tickets, simply list the status as is.
        } else {
            self.status.clone()
        }
    }
}

/// Extract the account name before `@` from an email address.
fn email_prefix(email: &str) -> &str {
    if let Some(prefix) = email.split('@').next() {
        prefix
    } else {
        email
    }
}

/// List the most common product set in the tickets.
fn most_common_product(tickets: &[AbstractTicket]) -> Option<&str> {
    let products: Counter<&str> = tickets
        .iter()
        .map(|ticket| ticket.product.as_str())
        .collect();

    products
        .k_most_common_ordered(1)
        .first()
        .map(|(elem, _frequency)| *elem)
}

#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Ord, PartialOrd)]
enum Version<'a> {
    Raw(&'a str),
    Parsed {
        x: u32,
        y: Option<u32>,
        z: Option<u32>,
    },
}

impl<'a> Version<'a> {
    /// Try to extract an x.y.z, x.y, or x version from a string.
    /// If no such version is found, return the original release string back.
    /// The intended purpose is to recognize release strings such as `rhel-9.3.0`,
    /// `9.3.0`, `9.3.0 Beta` and `v9.3.0` as identical versions and count them together.
    fn from(release: &'a str) -> Self {
        // Capture the regex:
        let caps_xyz = VERSION_XYZ_REGEX.captures(release);

        // Take x, y, and z groups from the capture:
        if let Some(caps) = caps_xyz {
            let x = extract_number(&caps, 1).expect("Regular expression failed.");
            let y = extract_number(&caps, 2);
            let z = extract_number(&caps, 3);

            return Version::Parsed { x, y, z };
        }

        let caps_xy = VERSION_XY_REGEX.captures(release);

        if let Some(caps) = caps_xy {
            let x = extract_number(&caps, 1).expect("Regular expression failed.");
            let y = extract_number(&caps, 2);
            let z = None;

            return Version::Parsed { x, y, z };
        }

        let caps_x = VERSION_X_REGEX.captures(release);

        if let Some(caps) = caps_x {
            let x = extract_number(&caps, 1).expect("Regular expression failed.");
            let y = None;
            let z = None;

            return Version::Parsed { x, y, z };
        }

        // All previous parsing failed. Return the original string.
        Version::Raw(release)
    }
}

impl fmt::Display for Version<'_> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            Self::Parsed { x, y, z } => {
                let mut s = x.to_string();
                if let Some(y) = y {
                    s = format!("{s}.{y}");
                }
                if let Some(z) = z {
                    s = format!("{s}.{z}");
                }
                write!(f, "{s}")
            }
            Self::Raw(s) => {
                write!(f, "{s}")
            }
        }
    }
}

fn extract_number(caps: &regex::Captures, index: usize) -> Option<u32> {
    caps.get(index)
        .map(|m| m.as_str())
        .and_then(|s| s.parse().ok())
}

/// List the most common release set in the tickets.
fn most_common_release(tickets: &[AbstractTicket]) -> Option<Version<'_>> {
    let mut releases: Counter<Version> = Counter::new();

    // Releases are a list, and each ticket can have several of them.
    // Update the counter with the values in the lists, rather than
    // with the lists themselves as values.
    for ticket in tickets {
        // Extract the x.y.z version numbers from the version strings.
        let extracted_versions = ticket
            .target_releases
            .iter()
            .map(|release| Version::from(release));
        // Count the x.y.z versions.
        releases.update(extracted_versions);
    }

    // Find the two most common versions.
    let two_versions: Vec<Version<'_>> = releases
        .k_most_common_ordered(2)
        .iter()
        .map(|(elem, _frequency)| *elem)
        .collect();

    log::debug!("The two most common versions: {:?}", two_versions);

    let first = two_versions.get(0);
    let second = two_versions.get(1);

    // In case the second most common version is more recent than the first most common, use the second one.
    // This might occur in a release that inherits many release notes from the previous release
    // and adds just a few more recent tickets on top.
    if second > first {
        log::info!(
            "The second most common version, {}, is greater than {}. Switching.",
            second.map(ToString::to_string).unwrap_or("None".into()),
            first.map(ToString::to_string).unwrap_or("None".into())
        );
        second.copied()
    } else {
        log::debug!(
            "The most common version, {first:?}, is greater than or equal to {second:?}. Keeping."
        );
        first.copied()
    }
}

/// All the data that the status table needs to render.
#[derive(Template, Serialize)] // this will generate the code...
#[template(path = "status-table.html")] // using the template in this path, relative
                                        // to the `templates` dir in the crate root
struct StatusTableTemplate<'a> {
    products: &'a str,
    release: &'a str,
    overall_progress: OverallProgress,
    tickets_with_checks: &'a [(&'a AbstractTicket, &'a Checks)],
    per_writer_stats: &'a [WriterStats<'a>],
    generated_date: &'a str,
}

/// Analyze all tickets and release notes, and produce a status table in two variants:
///
/// * As text with HTML markup.
/// * As a JSON map in text form.
pub fn analyze_status(tickets: &[AbstractTicket]) -> Result<(String, String)> {
    // Determine the product and release.
    let product = most_common_product(tickets);
    let release = most_common_release(tickets);
    let release_s = release.map(|r| r.to_string());

    // Display these placeholders if there are no products or releases at all.
    let releases_display = release_s.as_ref().map_or("no releases", |r| r.as_str());
    let products_display = product.unwrap_or("no releases");

    let date_today = OffsetDateTime::now_utc()
        .format(&Rfc2822)
        .expect("Cannot format the current date to RFC2822. This is a bug.");

    // Store checks in their own Vec and zip them with tickets by reference,
    // This satisfies ownership requirements, because the template
    // needs to receive both tickets and checks by reference.
    let checks: Vec<Checks> = tickets
        .iter()
        .map(|ticket| ticket.checks(release))
        .collect();
    let tickets_with_checks: Vec<(&AbstractTicket, &Checks)> =
        tickets.iter().zip(checks.iter()).collect();

    let overall_progress: OverallProgress = checks.as_slice().into();

    let writer_stats = calculate_writer_stats(&tickets_with_checks);

    let status_table = StatusTableTemplate {
        products: products_display,
        release: releases_display,
        overall_progress,
        per_writer_stats: &writer_stats,
        tickets_with_checks: &tickets_with_checks,
        generated_date: &date_today,
    };

    let as_html = status_table
        .render()
        .wrap_err("Failed to prepare the status table.")?;

    let as_json = serde_json::to_string(&status_table)
        .wrap_err("Failed to prepare the JSON status output.")?;

    Ok((as_html, as_json))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parse_versions() {
        let release = "rhel-9.3.0 Beta";
        let version_9_3_0 = Version::from(release);

        assert_eq!(
            version_9_3_0,
            Version::Parsed {
                x: 9,
                y: Some(3),
                z: Some(0)
            }
        );

        let release = "rhel-9.3";
        let version_9_3 = Version::from(release);

        assert_eq!(
            version_9_3,
            Version::Parsed {
                x: 9,
                y: Some(3),
                z: None
            }
        );

        let release = "RHEL 9";
        let version_9 = Version::from(release);

        assert_eq!(
            version_9,
            Version::Parsed {
                x: 9,
                y: None,
                z: None
            }
        );

        let release = "No version here.";
        let version_none = Version::from(release);

        assert_eq!(version_none, Version::Raw(release));
    }

    #[test]
    fn compare_versions() {
        let release = "rhel-9.3.0 Beta";
        let version_9_3_0 = Version::from(release);

        let release = "rhel-9.3";
        let version_9_3 = Version::from(release);

        let release = "RHEL 9";
        let version_9 = Version::from(release);

        let release = "rhel-8.9.1";
        let version_8_9_1 = Version::from(release);

        let release = "No version here.";
        let version_none = Version::from(release);

        assert!(version_9_3_0 > version_8_9_1);
        assert!(version_9_3_0 > version_9_3);
        assert!(version_9_3 > version_9);
        assert!(version_8_9_1 > version_none);
        assert!(version_9 > version_8_9_1);
    }
}