KiThe 0.3.6

//! # NIST Chemistry WebBook Parser Module
//!
//! ## Aim
//! This module provides automated web scraping capabilities for the NIST Chemistry WebBook
//! (https://webbook.nist.gov). It extracts thermodynamic data including heat capacity coefficients,
//! formation enthalpies, entropies, and molar masses directly from NIST web pages.
//!
//! ## Main Data Structures and Logic
//! - `NistParser<C>`: Generic parser with dependency injection for HTTP client (enables testing)
//! - `NistInput`: Structure storing parsed thermodynamic data in Shomate equation format
//! - `Phase` enum: Handles gas, liquid, and solid phase data extraction
//! - `SearchType` enum: Specifies which properties to extract (Cp, ΔH, ΔS, molar mass, or all)
//! - Uses CSS selectors to parse HTML tables and extract numerical data
//!
//! ## Key Methods
//! - `get_data()`: Main method orchestrating the complete data extraction process
//! - `construct_url()`: Builds appropriate NIST URLs based on substance identifier type
//! - `extract_cp()`: Parses Shomate equation coefficients from HTML tables
//! - `extract_thermodynamic_data()`: Extracts formation enthalpy and entropy values
//! - `extract_molar_mass()`: Parses molecular weight information
//! - `caclc_cp_dh_ds()`: Computes thermodynamic properties using Shomate equations
//!
//! ## Usage
//! ```rust, ignore
//! let parser = NistParser::new();
//! let data = parser.get_data("CH4", SearchType::All, Phase::Gas)?;
//! data.pretty_print();  // Display formatted results
//! let (cp, dh, ds) = data.caclc_cp_dh_ds(298.15)?;
//! ```
//!
//! ## Interesting Features
//! - Intelligent URL construction: detects CAS numbers, chemical formulas, or names
//! - Robust HTML parsing with CSS selectors for reliable data extraction
//! - Handles NIST's multi-page navigation automatically (search results → substance page → data page)
//! - Supports dependency injection for HTTP client (enables mocking in tests)
//! - Implements Shomate equation calculations with symbolic expression support
//! - Provides both numerical functions and symbolic expressions for integration
//! - Handles multiple temperature ranges with automatic coefficient selection
//! - Includes comprehensive error handling for network issues and parsing failures
//! - Creates pretty-printed tables for data visualization

use reqwest::blocking::{Client, Response};
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use thiserror::Error;
use url::Url;

const MAX_NIST_RESPONSE_BYTES: usize = 2 * 1024 * 1024;
/// HTTP client trait for dependency injection
pub trait HttpClient {
    fn get_response(&self, url: &str) -> Result<Response, reqwest::Error>;
}

// Implementation for the real reqwest client
impl HttpClient for Client {
    fn get_response(&self, url: &str) -> Result<Response, reqwest::Error> {
        self.get(url).send()
    }
}
/// error types for the reqwest client
#[derive(Debug, Error)]
#[allow(dead_code)]
pub enum NistError {
    #[error("NIST parser configuration error: {0}")]
    ConfigurationError(String),
    #[error("Network error: {0}")]
    NetworkError(#[from] reqwest::Error),
    #[error("Unexpected HTTP status: {0}")]
    HttpStatus(reqwest::StatusCode),
    #[error("URL parsing error: {0}")]
    UrlError(#[from] url::ParseError),
    #[error("NIST response exceeded the accepted size limit: {actual} bytes > {limit} bytes")]
    ResponseTooLarge { limit: usize, actual: usize },
    #[error("Batch fetch failed for all {attempts} request(s): {last_error}")]
    BatchAllFailed { attempts: usize, last_error: String },
    #[error("Substance not found")]
    SubstanceNotFound,
    #[error("Invalid data format")]
    InvalidDataFormat,
}

/// struct for the substance data parsed from the NIST web page
#[derive(Debug, Serialize, Deserialize, Clone)]
#[allow(non_snake_case)]
pub struct NistInput {
    pub cp: Option<Vec<Vec<f64>>>,
    pub T: Option<Vec<Vec<f64>>>,
    pub dh: Option<f64>,
    pub ds: Option<f64>,
    pub molar_mass: Option<f64>,
}
/// Phase enum: solid, liquid, gas
#[derive(Debug, Clone, Copy, PartialEq)]
#[allow(dead_code)]
pub enum Phase {
    Gas,
    Solid,
    Liquid,
}

#[derive(Debug, Clone, Copy, PartialEq)]
#[allow(dead_code)]
pub enum SearchType {
    Cp,
    DeltaH,
    DeltaS,
    MolarMass,
    All,
}

/// One item in a batch fetch report.
#[derive(Debug)]
pub struct NistBatchEntry {
    pub substance: String,
    pub search_type: SearchType,
    pub phase: Phase,
    pub result: Result<NistInput, NistError>,
}

/// Aggregated outcome for multiple NIST fetch requests.
#[derive(Debug, Default)]
pub struct NistBatchReport {
    pub entries: Vec<NistBatchEntry>,
}

impl NistBatchReport {
    pub fn successes(&self) -> usize {
        self.entries
            .iter()
            .filter(|entry| entry.result.is_ok())
            .count()
    }

    pub fn failures(&self) -> usize {
        self.entries
            .iter()
            .filter(|entry| entry.result.is_err())
            .count()
    }
}

impl Phase {
    pub fn as_str(&self) -> &'static str {
        match self {
            Phase::Gas => "gas",
            Phase::Solid => "solid",
            Phase::Liquid => "liquid",
        }
    }
}

pub struct NistParser<C: HttpClient> {
    client: C,
    base_url: Url,
}

impl NistParser<Client> {
    pub fn try_new() -> Result<Self, NistError> {
        let user_agent = format!("KiThe/{} NIST parser", env!("CARGO_PKG_VERSION"));
        let client = Client::builder()
            .connect_timeout(Duration::from_secs(5))
            .timeout(Duration::from_secs(20))
            .user_agent(user_agent)
            .build()
            .map_err(|err| NistError::ConfigurationError(err.to_string()))?;
        let base_url = Url::parse("https://webbook.nist.gov").map_err(NistError::UrlError)?;
        Ok(Self { client, base_url })
    }

    pub fn new() -> Self {
        Self::try_new().expect("failed to build configured NIST HTTP client")
    }
}

impl<C: HttpClient> NistParser<C> {
    #[allow(dead_code)]
    pub fn with_client(client: C) -> Self {
        Self {
            client,
            base_url: Url::parse("https://webbook.nist.gov")
                .expect("default NIST base URL must be valid"),
        }
    }

    #[allow(dead_code)]
    pub fn with_client_and_base_url(client: C, base_url: Url) -> Self {
        Self { client, base_url }
    }
    ///////////////////////////////////TRAVELLING THE WEBSITE: http://webbook.nist.gov/cgi/cbook.cgi///////////////////////////////////////////
    pub fn get_data(
        &self,
        substance: &str,
        search_type: SearchType,
        phase: Phase,
    ) -> Result<NistInput, NistError> {
        let url = self.construct_url(substance)?;
        let html = self.fetch_page(&url)?;
        if !self.check_substance_exists(&html) {
            return Err(NistError::SubstanceNotFound);
        }

        let url_of_substance = self.get_url_of_substance(&html, &url)?;
        let html_of_substance = self.fetch_page(&url_of_substance)?;

        let final_url = self.get_final_url(&html_of_substance, &url_of_substance, phase)?;
        let _html_of_phase = self.fetch_page(&final_url)?;

        let mut data = self.parse_data(&final_url, search_type, phase)?;

        // Set dh to 0.0 for simple substances if it's None
        if data.dh.is_none() && self.is_simple_substance(substance) {
            data.dh = Some(0.0);
        }

        Ok(data)
    }

    /// Fetch multiple substances and collect a per-item report.
    ///
    /// The method only fails when every request fails, so callers can still
    /// inspect partial success.
    pub fn get_data_batch(
        &self,
        requests: &[(String, SearchType, Phase)],
    ) -> Result<NistBatchReport, NistError> {
        let mut report = NistBatchReport::default();
        let mut last_error: Option<String> = None;

        for (substance, search_type, phase) in requests {
            let result = self.get_data(substance, *search_type, *phase);
            if let Err(err) = &result {
                last_error = Some(err.to_string());
            }
            report.entries.push(NistBatchEntry {
                substance: substance.clone(),
                search_type: *search_type,
                phase: *phase,
                result,
            });
        }

        if report.successes() == 0 {
            return Err(NistError::BatchAllFailed {
                attempts: requests.len(),
                last_error: last_error
                    .unwrap_or_else(|| "no batch requests were executed".to_string()),
            });
        }

        Ok(report)
    }

    pub fn construct_url(&self, substance: &str) -> Result<Url, NistError> {
        let substance = substance.replace(' ', "");
        let base = self.base_url.as_str().trim_end_matches('/');

        // Try to determine if it's a CAS number (contains '-')
        if substance.contains('-') {
            Ok(Url::parse(&format!(
                "{base}/cgi/cbook.cgi?ID={substance}&Units=SI"
            ))?)
        } else if substance.chars().any(|c| c.is_ascii_digit()) {
            // If contains numbers, assume it's a chemical formula
            Ok(Url::parse(&format!(
                "{base}/cgi/cbook.cgi?Formula={substance}&NoIon=on&Units=SI"
            ))?)
        } else {
            // Otherwise, assume it's a name
            Ok(Url::parse(&format!(
                "{base}/cgi/cbook.cgi?Name={substance}&Units=SI"
            ))?)
        }
    }

    pub(crate) fn fetch_page(&self, url: &Url) -> Result<String, NistError> {
        let response = self.client.get_response(url.as_str())?;
        let status = response.status();
        if !status.is_success() {
            return Err(NistError::HttpStatus(status));
        }

        if let Some(content_length) = response.content_length() {
            if content_length as usize > MAX_NIST_RESPONSE_BYTES {
                return Err(NistError::ResponseTooLarge {
                    limit: MAX_NIST_RESPONSE_BYTES,
                    actual: content_length as usize,
                });
            }
        }

        let html = response.text()?;
        if html.len() > MAX_NIST_RESPONSE_BYTES {
            return Err(NistError::ResponseTooLarge {
                limit: MAX_NIST_RESPONSE_BYTES,
                actual: html.len(),
            });
        }
        self.validate_html_content(&html)?;
        Ok(html)
    }

    fn validate_html_content(&self, html: &str) -> Result<(), NistError> {
        let trimmed = html.trim();
        if trimmed.is_empty() {
            return Err(NistError::InvalidDataFormat);
        }

        let lower = trimmed.to_ascii_lowercase();
        if !(lower.contains("<html") || lower.contains("<!doctype") || lower.contains("<body")) {
            return Err(NistError::InvalidDataFormat);
        }

        Ok(())
    }

    fn check_substance_exists(&self, html: &str) -> bool {
        let document = Html::parse_document(html);
        let selector = Selector::parse("h1").unwrap();

        for element in document.select(&selector) {
            let text = element.text().collect::<String>();
            if text.contains("Not Found") {
                return false;
            }
        }
        true
    }

    fn get_url_of_substance(&self, html: &str, original_url: &Url) -> Result<Url, NistError> {
        let document = Html::parse_document(html);

        // Check if we're on a search results page
        if let Ok(selector) = Selector::parse("ol li a") {
            if let Some(first_result) = document.select(&selector).next() {
                if let Some(href) = first_result.value().attr("href") {
                    return self.base_url.join(href).map_err(NistError::UrlError);
                }
            }
        }

        // If not on search results page, use original URL
        Ok(original_url.clone())
    }

    fn get_final_url(
        &self,
        html: &str,
        url_of_substance: &Url,
        phase: Phase,
    ) -> Result<Url, NistError> {
        let document = Html::parse_document(html);
        let link_text = match phase {
            Phase::Gas => "Gas phase thermochemistry data",
            Phase::Solid => "Condensed phase thermochemistry data",
            Phase::Liquid => "Condensed phase thermochemistry data",
        };

        let selector = Selector::parse("a").unwrap();
        for element in document.select(&selector) {
            if element.text().collect::<String>().contains(link_text) {
                if let Some(href) = element.value().attr("href") {
                    return url_of_substance
                        .join(href)
                        .map_err(|e| NistError::UrlError(e));
                }
            }
        }

        // If we couldn't find the link, return the original URL
        Ok(url_of_substance.clone())
    }

    /// Helper function to detect if a substance is a simple element
    fn is_simple_substance(&self, substance: &str) -> bool {
        let simple_substances = [
            "H2", "He", "Li", "Be", "B", "C", "N2", "O2", "F2", "Ne", "Na", "Mg", "Al", "Si", "P",
            "S", "Cl2", "Ar", "K", "Ca", "Sc", "Ti", "V", "Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn",
            "Ga", "Ge", "As", "Se", "Br2", "Kr", "Rb", "Sr", "Y", "Zr", "Nb", "Mo", "Tc", "Ru",
            "Rh", "Pd", "Ag", "Cd", "In", "Sn", "Sb", "Te", "I2", "Xe", "Cs", "Ba", "La", "Ce",
            "Pr", "Nd", "Pm", "Sm", "Eu", "Gd", "Tb", "Dy", "Ho", "Er", "Tm", "Yb", "Lu", "Hf",
            "Ta", "W", "Re", "Os", "Ir", "Pt", "Au", "Hg", "Tl", "Pb", "Bi", "Po", "At", "Rn",
            "Fr", "Ra", "Ac", "Th", "Pa", "U", "Np", "Pu", "Am", "Cm", "Bk", "Cf", "Es", "Fm",
            "Md", "No", "Lr", "Rf", "Db", "Sg", "Bh", "Hs", "Mt", "Ds", "Rg", "Cn", "Nh", "Fl",
            "Mc", "Lv", "Ts", "Og",
        ];
        simple_substances.contains(&substance)
    }

    ////////////////////////////////PARSING DATA//////////////////////////////////////////////////////////////
    fn parse_data(
        &self,
        url: &Url,
        search_type: SearchType,
        phase: Phase,
    ) -> Result<NistInput, NistError> {
        let html = self.fetch_page(url)?;
        let document = Html::parse_document(&html);
        //  println!("\n \n document: {:?}", document);

        let mut data = NistInput {
            cp: None,
            T: None,
            dh: None,
            ds: None,
            molar_mass: None,
        };

        match search_type {
            SearchType::Cp => {
                (data.cp, data.T) = self.extract_cp(&document, phase)?;
            }
            SearchType::DeltaH | SearchType::DeltaS => {
                let (dh, ds) = self.extract_thermodynamic_data(&document, phase)?;
                data.dh = dh;
                data.ds = ds;
            }
            SearchType::MolarMass => {
                data.molar_mass = self.extract_molar_mass(&document)?;
            }
            SearchType::All => {
                (data.cp, data.T) = self.extract_cp(&document, phase)?;
                let (dh, ds) = self.extract_thermodynamic_data(&document, phase)?;
                data.dh = dh;
                data.ds = ds;
                data.molar_mass = self.extract_molar_mass(&document)?;
            }
        }
        Ok(data)
    }

    fn extract_cp(
        &self,
        document: &Html,
        phase: Phase,
    ) -> Result<(Option<Vec<Vec<f64>>>, Option<Vec<Vec<f64>>>), NistError> {
        let table_selector = match phase {
            //matches the phase parameter to determine the appropriate CSS selector for the table.
            Phase::Gas => {
                Selector::parse("table[aria-label='Gas Phase Heat Capacity (Shomate Equation)']")
                    .unwrap()
            }
            Phase::Solid => {
                Selector::parse("table[aria-label='Solid Phase Heat Capacity (Shomate Equation)']")
                    .unwrap()
            }
            Phase::Liquid => {
                Selector::parse("table[aria-label='Liquid Phase Heat Capacity (Shomate Equation)']")
                    .unwrap()
            }
        }; //Selector::parse(...).unwrap() parses the CSS selector string and unwraps the result, assuming it is valid.

        if let Some(table) = document.select(&table_selector).next() {
            //attempt to select the first table element that
            //matches the table_selector from the document. document.select(&table_selector) returns an iterator over matching elements.

            #[allow(non_snake_case)]
            let mut headers_T: Vec<Vec<f64>> = Vec::new();
            // Parse headers
            if let Some(header_row) = table.select(&Selector::parse("tr").unwrap()).next() {
                headers_T = header_row
                    .select(&Selector::parse("td").unwrap())
                    .filter_map(|cell| {
                        let text = cell.text().collect::<String>();

                        let temps: Vec<f64> = text
                            .split_whitespace() // Split the string by whitespace
                            .filter_map(|s| s.parse::<f64>().ok()) // Parse each part to f32, ignoring any errors
                            .collect(); // Collect the results into a Vec<f32>

                        Some(temps)
                    })
                    .collect();
            }
            //    println!("\n \n headers: {:?} \n \n", headers_T);
            let mut coefficients: Vec<Vec<f64>> = vec![Vec::with_capacity(9); headers_T.len()];
            for row in table.select(&Selector::parse("tr").unwrap()).skip(1) {
                //skip the first row, which contains the header
                let cells: Vec<f64> = row
                    .select(&Selector::parse("td").unwrap())
                    .filter_map(|cell| {
                        cell.text()
                            .collect::<String>()
                            .split_whitespace()
                            .next()
                            .and_then(|s| s.parse::<f64>().ok())
                    })
                    .collect();

                if !cells.is_empty() {
                    for (i, cell) in cells.iter().enumerate() {
                        coefficients[i].push(*cell);
                    }
                }
            }

            if !coefficients.is_empty() && !headers_T.is_empty() {
                return Ok((Some(coefficients), Some(headers_T)));
            }
        }

        Ok((None, None))
    }

    fn extract_thermodynamic_data(
        &self,
        document: &Html,
        phase: Phase,
    ) -> Result<(Option<f64>, Option<f64>), NistError> {
        let table_selector = Selector::parse("table").unwrap();

        if let Some(table) = document.select(&table_selector).next() {
            let mut dh = None;
            let mut ds = None;

            for row in table.select(&Selector::parse("tr").unwrap()) {
                let cells: Vec<String> = row
                    .select(&Selector::parse("td").unwrap())
                    .map(|cell| cell.text().collect::<String>())
                    .collect();

                if cells.len() >= 2 {
                    //this is a numerical value of dH or dS
                    let value_str = cells[1].trim();
                    if let Some(value) = value_str.split('±').next() {
                        if cells[0].contains("H°")
                            && cells[0].contains("f")
                            && cells[0].contains(phase.as_str())
                        {
                            //    println!("\n \n value: {:?} \n \n", value);
                            if let Ok(val) = value.trim().parse::<f64>() {
                                dh = Some(val);
                            }
                        } else if cells[0].starts_with("S°") && cells[0].contains(phase.as_str()) {
                            if let Ok(val) = value.trim().parse::<f64>() {
                                ds = Some(val);
                            }
                        }
                    }
                }
            }

            Ok((dh, ds))
        } else {
            Ok((None, None))
        }
    }

    fn extract_molar_mass(&self, document: &Html) -> Result<Option<f64>, NistError> {
        let selector = Selector::parse("li").unwrap();

        for element in document.select(&selector) {
            let text = element.text().collect::<String>();
            if text.contains("Molecular weight") {
                if let Some(value) = text.split(':').nth(1) {
                    if let Ok(mass) = value.trim().parse::<f64>() {
                        return Ok(Some(mass));
                    }
                }
            }
        }

        Ok(None)
    }
}

impl NistInput {
    pub fn new() -> NistInput {
        NistInput {
            cp: None,
            T: None,
            dh: None,
            ds: None,
            molar_mass: None,
        }
    }

    pub fn extract_coefficients(
        &mut self,
        T: f64,
    ) -> Result<(f64, f64, f64, f64, f64, f64, f64, f64), NistError> {
        if let (Some(T_ranges), Some(cp_coeffs)) = (&self.T, &self.cp) {
            for (i, T_pairs) in T_ranges.iter().enumerate() {
                if T >= T_pairs[0] && T <= T_pairs[1] {
                    let coeffs = &cp_coeffs[i];
                    if coeffs.len() >= 8 {
                        return Ok((
                            coeffs[0], coeffs[1], coeffs[2], coeffs[3], coeffs[4], coeffs[5],
                            coeffs[6], coeffs[7],
                        ));
                    }
                }
            }
        }
        Err(NistError::InvalidDataFormat)
    }

    pub fn caclc_cp_dh_ds(&mut self, T: f64) -> Result<(f64, f64, f64), NistError> {
        let (a, b, c, d, e, f, g, h) = self.extract_coefficients(T)?;
        let t = T / 1000.0; // Convert to kK for NIST equations

        let cp = a + b * t + c * t.powi(2) + d * t.powi(3) + e / t.powi(2);

        let dh0 = self.dh.unwrap_or(0.0); // Use 0.0 if dh is None (simple substances)
        let dh = a * t + (b * t.powi(2)) / 2.0 + (c * t.powi(3)) / 3.0 + (d * t.powi(4)) / 4.0
            - e / t
            + f
            - h
            + dh0;

        let ds = a * t.ln() + b * t + (c * t.powi(2)) / 2.0 + (d * t.powi(3)) / 3.0
            - e / (2.0 * t.powi(2))
            + g;

        Ok((cp, dh, ds))
    }

    pub fn pretty_print(&self) {
        println!("NIST Data:");
        if let Some(molar_mass) = self.molar_mass {
            println!("  Molar mass: {} g/mol", molar_mass);
        }
        if let Some(dh) = self.dh {
            println!("  Formation enthalpy: {} kJ/mol", dh);
        } else {
            println!("  Formation enthalpy: 0.0 kJ/mol (simple substance)");
        }
        if let Some(ds) = self.ds {
            println!("  Standard entropy: {} J/mol·K", ds);
        }
        if let (Some(T_ranges), Some(cp_coeffs)) = (&self.T, &self.cp) {
            println!("  Heat capacity coefficients:");
            for (i, (T_range, coeffs)) in T_ranges.iter().zip(cp_coeffs.iter()).enumerate() {
                println!("    Range {}: {:.0}-{:.0} K", i + 1, T_range[0], T_range[1]);
                println!(
                    "      A={:.3e}, B={:.3e}, C={:.3e}, D={:.3e}",
                    coeffs[0], coeffs[1], coeffs[2], coeffs[3]
                );
                if coeffs.len() > 4 {
                    println!(
                        "      E={:.3e}, F={:.3e}, G={:.3e}, H={:.3e}",
                        coeffs[4], coeffs[5], coeffs[6], coeffs[7]
                    );
                }
            }
        }
    }
}

/*
TODO!
Added a HttpClient trait for dependency injection
Modified NistParser to be generic over the HTTP client type
Added comprehensive tests including:
Unit tests for URL construction
Tests for phase conversion
Tests for data structure initialization
Mock tests for HTTP requests
Error handling tests
The test suite now covers:
URL Construction:
CAS number URLs
Chemical formula URLs
Chemical name URLs
Space handling in names
Phase Conversion:
Gas phase string conversion
Solid phase string conversion
Real Substance Fetching:
Methane data fetch
Water data fetch
Thermodynamic data fetch
Mock Tests:
Successful substance fetch with mocked response
Not found error handling
Network error handling
Data Structure:
Initialization and access of  NistInput fields
*/