provenant/parsers/

about.rs

// SPDX-FileCopyrightText: nexB Inc. and others
// SPDX-FileCopyrightText: Provenant contributors
// SPDX-License-Identifier: Apache-2.0
// Derived from ScanCode Toolkit (Apache-2.0); modified. See NOTICE.

//! Parser for AboutCode .ABOUT metadata files.
//!
//! Extracts package metadata from AboutCode .ABOUT YAML files which describe
//! software components, licenses, and related information.
//!
//! # Supported Formats
//! - .ABOUT (case-sensitive uppercase extension)
//!
//! # Key Features
//! - YAML-based metadata parsing
//! - Package URL (purl) parsing for type/namespace extraction
//! - Owner party information
//! - File reference tracking (about_resource field)
//! - License expression extraction
//! - Flexible field mapping (home_url/homepage_url)
//!
//! # Implementation Notes
//! - Uses yaml_serde for YAML parsing
//! - Uses packageurl crate for purl parsing
//! - Extension is case-sensitive and must be uppercase (.ABOUT not .about)
//! - Type can be overridden by 'type' field or extracted from 'purl' field
//! - Graceful error handling: logs warnings and returns default on parse failure

use crate::models::{DatasourceId, FileReference, PackageData, PackageType, Party, PartyType};
use crate::parser_warn as warn;
use crate::parsers::utils::{read_file_to_string, truncate_field};
use packageurl::PackageUrl;
use std::path::Path;
use std::str::FromStr;
use url::Url;
use yaml_serde::Value;

use super::PackageParser;
use super::license_normalization::{
    DeclaredLicenseMatchMetadata, build_declared_license_data, normalize_spdx_declared_license,
    normalize_spdx_expression,
};

const FIELD_TYPE: &str = "type";
const FIELD_PURL: &str = "purl";
const FIELD_PACKAGE_URL: &str = "package_url";
const FIELD_NAMESPACE: &str = "namespace";
const FIELD_NAME: &str = "name";
const FIELD_VERSION: &str = "version";
const FIELD_HOME_URL: &str = "home_url";
const FIELD_HOMEPAGE_URL: &str = "homepage_url";
const FIELD_DOWNLOAD_URL: &str = "download_url";
const FIELD_COPYRIGHT: &str = "copyright";
const FIELD_LICENSE_EXPRESSION: &str = "license_expression";
const FIELD_OWNER: &str = "owner";
const FIELD_ABOUT_RESOURCE: &str = "about_resource";

/// AboutCode .ABOUT file parser.
///
/// Parses AboutCode metadata files that contain package information,
/// licensing, and file references in YAML format.
pub struct AboutFileParser;

#[derive(Clone)]
struct InferredAboutIdentity {
    package_type: PackageType,
    namespace: Option<String>,
    name: Option<String>,
    version: Option<String>,
}

impl PackageParser for AboutFileParser {
    const PACKAGE_TYPE: PackageType = PackageType::About;

    fn extract_packages(path: &Path) -> Vec<PackageData> {
        let yaml = match read_and_parse_yaml(path) {
            Ok(yaml) => yaml,
            Err(e) => {
                warn!("Failed to read or parse .ABOUT file at {:?}: {}", path, e);
                return vec![default_package_data()];
            }
        };

        // Extract type and purl information
        let about_type = yaml
            .get(FIELD_TYPE)
            .and_then(|v| v.as_str())
            .map(String::from);

        let about_namespace = yaml
            .get(FIELD_NAMESPACE)
            .and_then(|v| v.as_str())
            .map(|v| truncate_field(v.to_string()));

        let purl_string = yaml
            .get(FIELD_PURL)
            .and_then(|v| v.as_str())
            .or_else(|| yaml.get(FIELD_PACKAGE_URL).and_then(|v| v.as_str()))
            .map(|v| truncate_field(v.to_string()));

        // Parse purl if present
        let (purl_type, purl_namespace, purl_name, purl_version) =
            if let Some(ref purl_str) = purl_string {
                match PackageUrl::from_str(purl_str) {
                    Ok(purl) => (
                        Some(truncate_field(purl.ty().to_string())),
                        purl.namespace().map(|v| truncate_field(v.to_string())),
                        Some(truncate_field(purl.name().to_string())),
                        purl.version().map(|v| truncate_field(v.to_string())),
                    ),
                    Err(e) => {
                        warn!("Failed to parse purl '{}': {}", purl_str, e);
                        (None, None, None, None)
                    }
                }
            } else {
                (None, None, None, None)
            };

        let inferred = infer_about_from_download_url(
            yaml.get(FIELD_DOWNLOAD_URL).and_then(|v| v.as_str()),
            yaml.get(FIELD_NAME)
                .and_then(yaml_value_to_string)
                .as_deref(),
            yaml.get(FIELD_VERSION)
                .and_then(yaml_value_to_string)
                .as_deref(),
        );

        let explicit_package_type = about_type
            .clone()
            .and_then(|s| s.parse::<crate::models::PackageType>().ok());
        let parsed_purl_type = purl_type
            .clone()
            .and_then(|s| s.parse::<crate::models::PackageType>().ok());
        let has_parsed_purl_identity = parsed_purl_type.is_some()
            || purl_namespace.is_some()
            || purl_name.is_some()
            || purl_version.is_some();
        let inferred_identity = if explicit_package_type.is_none() && !has_parsed_purl_identity {
            inferred
        } else {
            None
        };

        let package_type = explicit_package_type
            .or(parsed_purl_type)
            .or_else(|| {
                inferred_identity
                    .as_ref()
                    .map(|identity| identity.package_type)
            })
            .unwrap_or(Self::PACKAGE_TYPE);

        // Priority: about_namespace > purl_namespace
        let namespace = about_namespace
            .clone()
            .or(purl_namespace.clone())
            .or_else(|| {
                inferred_identity
                    .as_ref()
                    .and_then(|identity| identity.namespace.clone())
            })
            .map(truncate_field);

        // Name and version from YAML or purl
        let name = yaml
            .get(FIELD_NAME)
            .and_then(yaml_value_to_string)
            .or(purl_name.clone())
            .or_else(|| {
                inferred_identity
                    .as_ref()
                    .and_then(|identity| identity.name.clone())
            })
            .map(truncate_field);

        let version = yaml
            .get(FIELD_VERSION)
            .and_then(yaml_value_to_string)
            .or(purl_version.clone())
            .or_else(|| {
                inferred_identity
                    .as_ref()
                    .and_then(|identity| identity.version.clone())
            })
            .map(truncate_field);

        // Homepage URL (two possible field names)
        let homepage_url = yaml
            .get(FIELD_HOME_URL)
            .and_then(|v| v.as_str())
            .or_else(|| yaml.get(FIELD_HOMEPAGE_URL).and_then(|v| v.as_str()))
            .map(|v| truncate_field(v.to_string()));

        let download_url = yaml
            .get(FIELD_DOWNLOAD_URL)
            .and_then(|v| v.as_str())
            .map(|v| truncate_field(v.to_string()));

        let copyright = yaml
            .get(FIELD_COPYRIGHT)
            .and_then(|v| v.as_str())
            .map(|v| truncate_field(v.to_string()));

        let extracted_license_statement = yaml
            .get(FIELD_LICENSE_EXPRESSION)
            .and_then(|v| v.as_str())
            .map(|v| truncate_field(v.to_string()));
        let file_references = extract_file_references(&yaml);
        let (declared_license_expression, declared_license_expression_spdx, license_detections) =
            extracted_license_statement
                .as_deref()
                .and_then(normalize_spdx_expression)
                .map(|normalized| {
                    build_declared_license_data(
                        normalized,
                        DeclaredLicenseMatchMetadata::single_line(
                            extracted_license_statement.as_deref().unwrap_or_default(),
                        ),
                    )
                })
                .unwrap_or_else(|| {
                    normalize_spdx_declared_license(extracted_license_statement.as_deref())
                });

        let vcs_url = yaml
            .get(Value::String("vcs_url".to_string()))
            .and_then(|v| v.as_str())
            .map(|v| truncate_field(v.to_string()));

        let extra_data = build_extra_data(&yaml);

        let purl = purl_string
            .or_else(|| {
                let name = yaml
                    .get(FIELD_NAME)
                    .and_then(yaml_value_to_string)
                    .or(purl_name.clone())
                    .or_else(|| {
                        inferred_identity
                            .as_ref()
                            .and_then(|identity| identity.name.clone())
                    });
                let version = yaml
                    .get(FIELD_VERSION)
                    .and_then(yaml_value_to_string)
                    .or(purl_version.clone())
                    .or_else(|| {
                        inferred_identity
                            .as_ref()
                            .and_then(|identity| identity.version.clone())
                    });
                let namespace = about_namespace.clone().or_else(|| {
                    inferred_identity
                        .as_ref()
                        .and_then(|identity| identity.namespace.clone())
                });
                build_about_purl(
                    package_type,
                    namespace.as_deref(),
                    name.as_deref(),
                    version.as_deref(),
                )
            })
            .map(truncate_field);

        // Owner party
        let parties = extract_owner_party(&yaml);

        // File references
        vec![PackageData {
            package_type: Some(package_type),
            namespace,
            name,
            version,
            qualifiers: None,
            subpath: None,
            primary_language: None,
            description: None,
            release_date: None,
            parties,
            keywords: Vec::new(),
            homepage_url,
            download_url,
            size: None,
            sha1: None,
            md5: None,
            sha256: None,
            sha512: None,
            bug_tracking_url: None,
            code_view_url: None,
            vcs_url,
            copyright,
            holder: None,
            declared_license_expression,
            declared_license_expression_spdx,
            license_detections,
            other_license_expression: None,
            other_license_expression_spdx: None,
            other_license_detections: Vec::new(),
            extracted_license_statement,
            notice_text: None,
            source_packages: Vec::new(),
            file_references,
            is_private: false,
            is_virtual: false,
            extra_data,
            dependencies: Vec::new(),
            repository_homepage_url: None,
            repository_download_url: None,
            api_data_url: None,
            datasource_id: Some(DatasourceId::AboutFile),
            purl,
        }]
    }

    fn is_match(path: &Path) -> bool {
        path.extension()
            .and_then(|ext| ext.to_str())
            .is_some_and(|ext| ext == "ABOUT")
    }

    fn metadata() -> Vec<super::metadata::ParserMetadata> {
        vec![super::metadata::ParserMetadata {
            description: "AboutCode .ABOUT metadata file",
            file_patterns: &["**/*.ABOUT"],
            package_type: "about",
            primary_language: "",
            documentation_url: Some(
                "https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html",
            ),
        }]
    }
}

/// Reads and parses a YAML file.
fn read_and_parse_yaml(path: &Path) -> Result<yaml_serde::Mapping, String> {
    let content =
        read_file_to_string(path, None).map_err(|e| format!("Failed to read file: {}", e))?;

    parse_yaml_mapping(&content)
        .or_else(|yaml_error| parse_shallow_scalar_mapping(&content).ok_or(yaml_error))
}

fn parse_yaml_mapping(content: &str) -> Result<yaml_serde::Mapping, String> {
    let value: Value =
        yaml_serde::from_str(content).map_err(|e| format!("Failed to parse YAML: {}", e))?;

    match value {
        Value::Mapping(map) => Ok(map),
        _ => Err("Expected YAML mapping at root".to_string()),
    }
}

fn parse_shallow_scalar_mapping(content: &str) -> Option<yaml_serde::Mapping> {
    let mut map = yaml_serde::Mapping::new();
    let mut saw_mapping_entry = false;

    for line in content.lines() {
        let trimmed = line.trim();
        if trimmed.is_empty() || trimmed.starts_with('#') {
            continue;
        }
        if line.starts_with(char::is_whitespace) {
            return None;
        }

        let (raw_key, raw_value) = trimmed.split_once(':')?;
        let key = raw_key.trim();
        if key.is_empty()
            || !key.chars().all(|character| {
                character.is_ascii_alphanumeric() || matches!(character, '_' | '-')
            })
        {
            return None;
        }

        let value = raw_value.trim();
        if value.is_empty() {
            return None;
        }

        saw_mapping_entry = true;
        map.insert(
            Value::String(key.to_string()),
            Value::String(unquote_yaml_scalar(value)),
        );
    }

    saw_mapping_entry.then_some(map)
}

fn unquote_yaml_scalar(value: &str) -> String {
    if value.len() >= 2 {
        let mut characters = value.chars();
        let first = characters.next();
        let last = value.chars().last();
        if matches!(
            (first, last),
            (Some('"'), Some('"')) | (Some('\''), Some('\''))
        ) {
            return value[1..value.len() - 1].to_string();
        }
    }
    value.to_string()
}

/// Converts a YAML value to a string, handling strings, numbers, and booleans.
fn yaml_value_to_string(value: &Value) -> Option<String> {
    match value {
        Value::String(s) => Some(s.clone()),
        Value::Number(n) => Some(n.to_string()),
        Value::Bool(b) => Some(b.to_string()),
        _ => None,
    }
}

/// Extracts owner party information from YAML.
fn extract_owner_party(yaml: &yaml_serde::Mapping) -> Vec<Party> {
    let owner = yaml
        .get(Value::String(FIELD_OWNER.to_string()))
        .map(|v| match v {
            Value::String(s) => truncate_field(s.clone()),
            _ => truncate_field(format!("{:?}", v)),
        });

    if let Some(owner_name) = owner {
        if !owner_name.is_empty() {
            vec![Party {
                r#type: Some(PartyType::Person),
                role: Some("owner".to_string()),
                name: Some(owner_name),
                email: None,
                url: None,
                organization: None,
                organization_url: None,
                timezone: None,
            }]
        } else {
            Vec::new()
        }
    } else {
        Vec::new()
    }
}

/// Extracts file references from YAML.
fn extract_file_references(yaml: &yaml_serde::Mapping) -> Vec<FileReference> {
    let about_resource = yaml
        .get(Value::String(FIELD_ABOUT_RESOURCE.to_string()))
        .and_then(|v| v.as_str());
    let license_file = yaml
        .get(Value::String("license_file".to_string()))
        .and_then(|v| v.as_str());
    let notice_file = yaml
        .get(Value::String("notice_file".to_string()))
        .and_then(|v| v.as_str());

    let mut refs = Vec::new();

    if let Some(path) = about_resource {
        refs.push(FileReference {
            path: truncate_field(path.to_string()),
            size: None,
            sha1: None,
            md5: None,
            sha256: None,
            sha512: None,
            extra_data: None,
        });
    }

    for path in [license_file, notice_file].into_iter().flatten() {
        refs.push(FileReference {
            path: truncate_field(path.to_string()),
            size: None,
            sha1: None,
            md5: None,
            sha256: None,
            sha512: None,
            extra_data: None,
        });
    }

    refs
}

/// Returns a default (empty) PackageData structure.
fn default_package_data() -> PackageData {
    PackageData {
        package_type: Some(PackageType::About),
        datasource_id: Some(DatasourceId::AboutFile),
        ..Default::default()
    }
}

fn infer_about_from_download_url(
    download_url: Option<&str>,
    about_name: Option<&str>,
    about_version: Option<&str>,
) -> Option<InferredAboutIdentity> {
    let url = Url::parse(download_url?).ok()?;
    let host = url.host_str()?;

    if matches!(host, "pypi.python.org" | "files.pythonhosted.org") {
        let name = about_name.map(str::to_string)?;
        let version = about_version.map(str::to_string);
        return Some(InferredAboutIdentity {
            package_type: PackageType::Pypi,
            namespace: None,
            name: Some(name),
            version,
        });
    }

    if matches!(host, "raw.githubusercontent.com" | "github.com") {
        let mut segments = url.path_segments()?;
        let owner = segments.next()?.to_string();
        let repo = segments.next()?.to_string();
        return Some(InferredAboutIdentity {
            package_type: PackageType::Github,
            namespace: Some(owner),
            name: Some(repo),
            version: None,
        });
    }

    None
}

fn build_about_purl(
    package_type: PackageType,
    namespace: Option<&str>,
    name: Option<&str>,
    version: Option<&str>,
) -> Option<String> {
    if package_type == PackageType::About {
        return None;
    }

    let name = name?;
    let mut purl = PackageUrl::new(package_type.as_str(), name).ok()?;
    if let Some(namespace) = namespace {
        purl.with_namespace(namespace).ok()?;
    }
    if let Some(version) = version {
        purl.with_version(version).ok()?;
    }
    Some(purl.to_string())
}

fn build_extra_data(
    yaml: &yaml_serde::Mapping,
) -> Option<std::collections::HashMap<String, serde_json::Value>> {
    let mut extra_data = std::collections::HashMap::new();
    for key in ["license_file", "notice_file", "notes"] {
        if let Some(value) = yaml.get(Value::String(key.to_string()))
            && let Some(value) = yaml_value_to_string(value)
        {
            extra_data.insert(
                key.to_string(),
                serde_json::Value::String(truncate_field(value)),
            );
        }
    }
    (!extra_data.is_empty()).then_some(extra_data)
}