arch_toolkit/index/

fetch.rs

//! Official repository index fetching functions for the index module.

use std::process::{Command, Stdio};

use crate::error::{ArchToolkitError, Result};
use crate::types::index::{OfficialIndex, OfficialPackage};

#[cfg(feature = "aur")]
use crate::client::{ArchClient, rate_limit_archlinux};

/// Default repositories queried when the caller does not supply a repo list.
const DEFAULT_REPOS: [&str; 3] = ["core", "extra", "multilib"];

/// What: Fetch the official package index using `pacman -Sl`.
///
/// Inputs:
/// - None: Attempts to fetch via `pacman -Sl` command.
///
/// Output:
/// - `Ok(OfficialIndex)` containing all official packages with name index rebuilt.
/// - `Err` if pacman is unavailable or output cannot be parsed.
///
/// Details:
/// - Uses `pacman -Sl` for fast, local fetching (no network required).
/// - Queries the default repositories (core, extra, multilib) only. On
///   derivative distros (`EndeavourOS`, `CachyOS`, ...) combine
///   [`detect_enabled_repos`] with [`fetch_official_index_for_repos`] instead.
/// - For API fallback, use `fetch_official_index_async()` instead.
/// - Rebuilds name index after fetching for O(1) lookups.
///
/// # Errors
///
/// - Returns `Err(ArchToolkitError::Parse)` if pacman is unavailable or output cannot be parsed.
///
/// # Example
///
/// ```no_run
/// use arch_toolkit::index::fetch_official_index;
///
/// let index = fetch_official_index()?;
/// println!("Found {} official packages", index.pkgs.len());
/// # Ok::<(), arch_toolkit::error::ArchToolkitError>(())
/// ```
pub fn fetch_official_index() -> Result<OfficialIndex> {
    fetch_via_pacman(&DEFAULT_REPOS)
}

/// What: Fetch the official package index for an explicit repository list.
///
/// Inputs:
/// - `repos`: Repository names to query via `pacman -Sl <repo>`, in order.
///
/// Output:
/// - `Ok(OfficialIndex)` containing packages from the given repositories.
/// - `Err` if pacman is unavailable or a repository query fails.
///
/// Details:
/// - Lets callers include derivative-distro repositories (`EndeavourOS`,
///   `CachyOS`, Chaotic-AUR, ...) that the default list omits; discover them
///   with [`detect_enabled_repos`].
/// - Deduplicates by `(repo, name)` and rebuilds the name index, exactly like
///   [`fetch_official_index`].
///
/// # Errors
///
/// - Returns `Err(ArchToolkitError::Parse)` if pacman is unavailable, a listed
///   repository is unknown to pacman, or output cannot be parsed.
///
/// # Example
///
/// ```no_run
/// use arch_toolkit::index::{detect_enabled_repos, fetch_official_index_for_repos};
///
/// let repos = detect_enabled_repos();
/// let repo_refs: Vec<&str> = repos.iter().map(String::as_str).collect();
/// let index = fetch_official_index_for_repos(&repo_refs)?;
/// println!("Found {} packages across {} repos", index.pkgs.len(), repos.len());
/// # Ok::<(), arch_toolkit::error::ArchToolkitError>(())
/// ```
pub fn fetch_official_index_for_repos(repos: &[&str]) -> Result<OfficialIndex> {
    fetch_via_pacman(repos)
}

/// What: Discover repositories enabled in `/etc/pacman.conf`.
///
/// Inputs:
/// - None: Reads the system pacman configuration.
///
/// Output:
/// - Repository names in declaration order (e.g., `["core", "extra", "multilib", "chaotic-aur"]`).
/// - The default list (core, extra, multilib) when the file cannot be read.
///
/// Details:
/// - Parses `[section]` headers, skipping `[options]`, and follows top-level
///   `Include =` directives one level deep (with simple `*` glob support) so
///   repos declared in included files are found too.
/// - Purely local file parsing; never invokes pacman or the network.
#[must_use]
pub fn detect_enabled_repos() -> Vec<String> {
    detect_enabled_repos_from(std::path::Path::new("/etc/pacman.conf"))
}

/// What: Discover repositories enabled in a specific pacman configuration file.
///
/// Inputs:
/// - `path`: Path to a pacman.conf-style file.
///
/// Output:
/// - Repository names in declaration order; the default list (core, extra,
///   multilib) when the file cannot be read.
///
/// Details:
/// - Same parsing rules as [`detect_enabled_repos`]; exists so callers and
///   tests can target non-system configuration files.
#[must_use]
pub fn detect_enabled_repos_from(path: &std::path::Path) -> Vec<String> {
    let Ok(content) = std::fs::read_to_string(path) else {
        tracing::debug!(path = %path.display(), "pacman.conf unreadable; using default repos");
        return DEFAULT_REPOS.iter().map(ToString::to_string).collect();
    };

    let mut repos: Vec<String> = Vec::new();
    collect_repo_sections(&content, &mut repos, true);
    if repos.is_empty() {
        return DEFAULT_REPOS.iter().map(ToString::to_string).collect();
    }
    repos
}

/// What: Collect repository section names from pacman.conf content.
///
/// Inputs:
/// - `content`: File content to scan.
/// - `repos`: Accumulator preserving declaration order without duplicates.
/// - `follow_includes`: Follow `Include =` directives (one level deep).
///
/// Details:
/// - `[options]` is skipped; comment lines (`#`) are ignored.
/// - Include values support a trailing `*` glob within a single directory,
///   matching pacman's common `Include = /etc/pacman.d/*.conf` usage.
fn collect_repo_sections(content: &str, repos: &mut Vec<String>, follow_includes: bool) {
    for line in content.lines() {
        let line = line.trim();
        if line.starts_with('#') {
            continue;
        }
        if let Some(section) = line.strip_prefix('[').and_then(|s| s.strip_suffix(']')) {
            let section = section.trim();
            if !section.is_empty()
                && !section.eq_ignore_ascii_case("options")
                && !repos.iter().any(|r| r == section)
            {
                repos.push(section.to_string());
            }
        } else if follow_includes && let Some(value) = line.strip_prefix("Include") {
            let Some(include_path) = value.split('=').nth(1).map(str::trim) else {
                continue;
            };
            for file in expand_include_glob(include_path) {
                if let Ok(included) = std::fs::read_to_string(&file) {
                    collect_repo_sections(&included, repos, false);
                }
            }
        }
    }
}

/// What: Expand a pacman.conf `Include` value into concrete file paths.
///
/// Inputs:
/// - `pattern`: Literal path or a pattern with `*` in the file-name component.
///
/// Output:
/// - Matching paths, sorted for deterministic ordering; the literal path when
///   no glob character is present.
///
/// Details:
/// - Only file-name globs are supported (e.g., `/etc/pacman.d/*.conf`), which
///   covers pacman's common usage without pulling in a glob dependency.
fn expand_include_glob(pattern: &str) -> Vec<std::path::PathBuf> {
    let path = std::path::Path::new(pattern);
    let Some(file_pattern) = path.file_name().and_then(|f| f.to_str()) else {
        return Vec::new();
    };
    if !file_pattern.contains('*') {
        return vec![path.to_path_buf()];
    }
    let Some(parent) = path.parent() else {
        return Vec::new();
    };
    let (prefix, suffix) = file_pattern.split_once('*').unwrap_or((file_pattern, ""));
    let Ok(entries) = std::fs::read_dir(parent) else {
        return Vec::new();
    };
    let mut matches: Vec<std::path::PathBuf> = entries
        .filter_map(std::result::Result::ok)
        .map(|e| e.path())
        .filter(|p| {
            p.file_name()
                .and_then(|f| f.to_str())
                .is_some_and(|name| name.starts_with(prefix) && name.ends_with(suffix))
        })
        .collect();
    matches.sort();
    matches
}

/// What: Fetch the official package index asynchronously, trying pacman first and falling back to API.
///
/// Inputs:
/// - None: Attempts to fetch via `pacman -Sl` first, then falls back to Arch Packages API.
///
/// Output:
/// - `Result<OfficialIndex>` containing all official packages with name index rebuilt.
///
/// Details:
/// - Tries `pacman -Sl` first (fast, local, no network required).
/// - Falls back to Arch Packages API if pacman is unavailable or fails.
/// - API method requires `aur` feature and network access.
/// - Rebuilds name index after fetching for O(1) lookups.
///
/// # Errors
///
/// - Returns `Err(ArchToolkitError::Parse)` if API fetch fails and pacman is unavailable.
///
/// # Example
///
/// ```no_run
/// use arch_toolkit::index::fetch_official_index_async;
///
/// # async fn example() -> Result<(), arch_toolkit::error::ArchToolkitError> {
/// let index = fetch_official_index_async().await?;
/// println!("Found {} official packages", index.pkgs.len());
/// # Ok(())
/// # }
/// ```
#[cfg(feature = "index")]
pub async fn fetch_official_index_async() -> Result<OfficialIndex> {
    // Try pacman first (fast, local)
    match tokio::task::spawn_blocking(|| fetch_via_pacman(&DEFAULT_REPOS))
        .await
        .map_err(|e| ArchToolkitError::Parse(format!("Blocking task failed: {e}")))?
    {
        Ok(index) => {
            tracing::debug!("Successfully fetched official index via pacman");
            return Ok(index);
        }
        Err(e) => {
            tracing::debug!("Failed to fetch via pacman: {}, falling back to API", e);
        }
    }

    // Fallback to API if pacman unavailable
    #[cfg(feature = "aur")]
    {
        let client = crate::client::ArchClient::new()
            .map_err(|e| ArchToolkitError::Parse(format!("Failed to create HTTP client: {e}")))?;
        fetch_via_api(&client).await
    }

    #[cfg(not(feature = "aur"))]
    {
        Err(ArchToolkitError::Parse(
            "pacman unavailable and API fetch requires 'aur' feature".to_string(),
        ))
    }
}

/// What: Fetch the official package index for an explicit repository list, asynchronously.
///
/// Inputs:
/// - `repos`: Repository names to query via `pacman -Sl <repo>`, in order.
///
/// Output:
/// - `Result<OfficialIndex>` containing packages from the given repositories.
///
/// Details:
/// - Pacman-only: unlike [`fetch_official_index_async`], this never falls back
///   to the network API, so behavior is predictable for offline-first callers.
/// - Runs the blocking pacman queries via `tokio::task::spawn_blocking`.
///
/// # Errors
///
/// - Returns `Err(ArchToolkitError::Parse)` if pacman is unavailable, a listed
///   repository is unknown to pacman, or the blocking task fails.
///
/// # Example
///
/// ```no_run
/// use arch_toolkit::index::fetch_official_index_for_repos_async;
///
/// # async fn example() -> Result<(), arch_toolkit::error::ArchToolkitError> {
/// let repos = vec!["core".to_string(), "extra".to_string(), "chaotic-aur".to_string()];
/// let index = fetch_official_index_for_repos_async(repos).await?;
/// println!("Found {} packages", index.pkgs.len());
/// # Ok(())
/// # }
/// ```
#[cfg(feature = "index")]
pub async fn fetch_official_index_for_repos_async(repos: Vec<String>) -> Result<OfficialIndex> {
    tokio::task::spawn_blocking(move || {
        let repo_refs: Vec<&str> = repos.iter().map(String::as_str).collect();
        fetch_via_pacman(&repo_refs)
    })
    .await
    .map_err(|e| ArchToolkitError::Parse(format!("Blocking task failed: {e}")))?
}

/// What: Fetch official packages using `pacman -Sl` command.
///
/// Inputs:
/// - `repos`: Repository names to query, in order.
///
/// Output:
/// - `Ok(OfficialIndex)` with packages from pacman output, deduplicated and indexed.
/// - `Err` if pacman command fails or output cannot be parsed.
///
/// Details:
/// - Executes `pacman -Sl <repo>` for each given repository.
/// - Parses output format: `"repo pkgname version [installed]"`.
/// - Deduplicates packages by `(repo, name)` tuple.
/// - Rebuilds name index after fetching.
/// - Sets `LC_ALL=C` and `LANG=C` for consistent locale-independent output.
///
/// # Errors
///
/// - Returns `Err(ArchToolkitError::Parse)` if pacman is unavailable or output cannot be parsed.
fn fetch_via_pacman(repos: &[&str]) -> Result<OfficialIndex> {
    let mut pkgs = Vec::new();

    for repo in repos {
        tracing::debug!("Running: pacman -Sl {}", repo);
        let output = Command::new("pacman")
            .args(["-Sl", repo])
            .env("LC_ALL", "C")
            .env("LANG", "C")
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .output()
            .map_err(|e| {
                ArchToolkitError::Parse(format!("Failed to execute pacman -Sl {repo}: {e}"))
            })?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(ArchToolkitError::Parse(format!(
                "pacman -Sl {repo} failed: {stderr}"
            )));
        }

        let text = String::from_utf8_lossy(&output.stdout);
        for line in text.lines() {
            // Format: "repo pkgname version [installed]"
            let mut parts = line.split_whitespace();
            let Some(repo_part) = parts.next() else {
                continue;
            };
            let Some(name) = parts.next() else {
                continue;
            };
            let version = parts.next().unwrap_or("");

            // Verify repo matches expected (sanity check)
            if repo_part != *repo {
                continue;
            }

            pkgs.push(OfficialPackage {
                name: name.to_string(),
                repo: repo_part.to_string(),
                arch: String::new(), // Not available from -Sl
                version: version.to_string(),
                description: String::new(), // Not available from -Sl
            });
        }
    }

    // Deduplicate by (repo, name)
    pkgs.sort_by(|a, b| a.repo.cmp(&b.repo).then(a.name.cmp(&b.name)));
    pkgs.dedup_by(|a, b| a.repo == b.repo && a.name == b.name);

    let mut index = OfficialIndex {
        pkgs,
        name_to_idx: std::collections::HashMap::new(),
    };
    index.rebuild_name_index();

    tracing::debug!("Fetched {} packages via pacman", index.pkgs.len());
    Ok(index)
}

/// What: Fetch official packages from Arch Packages API.
///
/// Inputs:
/// - `client`: HTTP client for making requests (must have `aur` feature enabled).
///
/// Output:
/// - `Ok(OfficialIndex)` with packages from API, deduplicated and indexed.
/// - `Err` if API requests fail or responses cannot be parsed.
///
/// Details:
/// - Fetches from `https://archlinux.org/packages/search/json/` endpoint.
/// - Paginates through all results for each repository (core, extra, multilib).
/// - Parses JSON response structure with package metadata.
/// - Uses rate limiting via `rate_limit_archlinux()`.
/// - Deduplicates packages by `(repo, name)` tuple.
/// - Rebuilds name index after fetching.
///
/// # Errors
///
/// - Returns `Err(ArchToolkitError::Parse)` if HTTP requests fail or response structure is invalid.
/// - Returns `Err(ArchToolkitError::Json)` if JSON parsing fails.
#[cfg(feature = "aur")]
async fn fetch_via_api(client: &ArchClient) -> Result<OfficialIndex> {
    let repos = ["core", "extra", "multilib"];
    let archs = ["x86_64", "any"];
    let limit = 250; // API limit per page
    let mut pkgs = Vec::new();

    for repo in &repos {
        for arch in &archs {
            let mut page = 1;
            let mut has_more = true;

            while has_more {
                let url = format!(
                    "https://archlinux.org/packages/search/json/?repo={repo}&arch={arch}&limit={limit}&page={page}"
                );

                tracing::debug!(
                    repo = repo,
                    arch = arch,
                    page = page,
                    "Fetching package page from API"
                );

                // Apply rate limiting
                let _permit = rate_limit_archlinux().await;

                let response = client.http_client().get(&url).send().await.map_err(|e| {
                    ArchToolkitError::Parse(format!(
                        "Failed to fetch packages from API (repo={repo}, arch={arch}, page={page}): {e}"
                    ))
                })?;

                let status = response.status();
                if !status.is_success() {
                    return Err(ArchToolkitError::Parse(format!(
                        "API returned error status {status} for repo={repo}, arch={arch}, page={page}"
                    )));
                }

                let json: serde_json::Value = response.json().await.map_err(|e| {
                    ArchToolkitError::Parse(format!("Failed to parse JSON response: {e}"))
                })?;

                // Parse results array
                let results = json
                    .get("results")
                    .and_then(|v| v.as_array())
                    .ok_or_else(|| {
                        ArchToolkitError::Parse(format!(
                            "Invalid API response: missing 'results' array for repo={repo}, arch={arch}, page={page}"
                        ))
                    })?;

                for result in results {
                    let pkgname =
                        result
                            .get("pkgname")
                            .and_then(|v| v.as_str())
                            .ok_or_else(|| {
                                ArchToolkitError::Parse(
                                    "Invalid API response: missing 'pkgname' field".to_string(),
                                )
                            })?;

                    let repo_name = result.get("repo").and_then(|v| v.as_str()).unwrap_or(repo);

                    let arch_name = result.get("arch").and_then(|v| v.as_str()).unwrap_or(arch);

                    let version = result
                        .get("pkgver")
                        .and_then(|v| v.as_str())
                        .map(|v| {
                            let rel = result.get("pkgrel").and_then(|r| r.as_str()).unwrap_or("");
                            if rel.is_empty() {
                                v.to_string()
                            } else {
                                format!("{v}-{rel}")
                            }
                        })
                        .unwrap_or_default();

                    let description = result
                        .get("pkgdesc")
                        .and_then(|v| v.as_str())
                        .unwrap_or_default()
                        .to_string();

                    pkgs.push(OfficialPackage {
                        name: pkgname.to_string(),
                        repo: repo_name.to_string(),
                        arch: arch_name.to_string(),
                        version,
                        description,
                    });
                }

                // Check if there are more pages
                let num_pages = json
                    .get("num_pages")
                    .and_then(serde_json::Value::as_u64)
                    .unwrap_or(1);
                has_more = page < num_pages;
                page += 1;
            }
        }
    }

    // Deduplicate by (repo, name)
    pkgs.sort_by(|a, b| a.repo.cmp(&b.repo).then(a.name.cmp(&b.name)));
    pkgs.dedup_by(|a, b| a.repo == b.repo && a.name == b.name);

    let mut index = OfficialIndex {
        pkgs,
        name_to_idx: std::collections::HashMap::new(),
    };
    index.rebuild_name_index();

    tracing::debug!("Fetched {} packages via API", index.pkgs.len());
    Ok(index)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    /// What: Verify `fetch_via_pacman` parses pacman output correctly.
    ///
    /// Inputs:
    /// - Mock pacman output with valid format.
    ///
    /// Output:
    /// - `OfficialIndex` with parsed packages, deduplicated.
    ///
    /// Details:
    /// - Tests parsing of pacman -Sl output format.
    /// - Tests deduplication logic.
    fn fetch_via_pacman_parses_output() {
        // This test would require mocking pacman command, which is complex
        // Instead, we test the parsing logic indirectly via integration tests
        // For unit tests, we verify the function exists and can be called
        let result = fetch_via_pacman(&DEFAULT_REPOS);
        // Result depends on system state (pacman may or may not be available)
        // We just verify it doesn't panic and returns a Result
        if let Ok(index) = result {
            assert!(!index.pkgs.is_empty() || index.pkgs.is_empty()); // Always true, just checking structure
        } else {
            // Pacman unavailable, which is acceptable
        }
    }

    #[test]
    /// What: Verify `detect_enabled_repos_from` parses section headers and skips `[options]`.
    ///
    /// Inputs:
    /// - Temporary pacman.conf with options, standard repos, and a derivative repo.
    ///
    /// Output:
    /// - Repo names in declaration order, without `options`, without duplicates.
    ///
    /// Details:
    /// - Also verifies the default-list fallback for unreadable paths.
    fn detect_enabled_repos_parses_sections() {
        let dir = std::env::temp_dir().join("arch-toolkit-test-pacmanconf");
        std::fs::create_dir_all(&dir).expect("create temp dir");
        let conf = dir.join("pacman.conf");
        std::fs::write(
            &conf,
            "# comment\n[options]\nHoldPkg = pacman\n\n[core]\nInclude = /nonexistent/mirrorlist\n[extra]\n[multilib]\n[chaotic-aur]\n[core]\n",
        )
        .expect("write conf");

        let repos = detect_enabled_repos_from(&conf);
        assert_eq!(repos, ["core", "extra", "multilib", "chaotic-aur"]);

        let missing = detect_enabled_repos_from(std::path::Path::new("/nonexistent/pacman.conf"));
        assert_eq!(missing, DEFAULT_REPOS.map(String::from));

        std::fs::remove_dir_all(&dir).ok();
    }

    #[test]
    /// What: Verify `fetch_official_index` fallback logic.
    ///
    /// Inputs:
    /// - Function call when pacman may or may not be available.
    ///
    /// Output:
    /// - Either pacman result or API result (if aur feature enabled).
    ///
    /// Details:
    /// - Tests that function attempts pacman first.
    /// - Tests graceful fallback to API if pacman unavailable.
    fn fetch_official_index_fallback() {
        let result = fetch_official_index();
        // Result depends on system state
        // We just verify it returns a Result and doesn't panic
        match result {
            Ok(index) => {
                // Success - either from pacman or API
                assert!(index.pkgs.is_empty() || !index.pkgs.is_empty());
            }
            Err(e) => {
                // Both methods failed, which is acceptable in test environment
                // Error should be descriptive
                let error_msg = format!("{e}");
                assert!(!error_msg.is_empty());
            }
        }
    }

    #[cfg(feature = "index")]
    #[tokio::test]
    /// What: Verify `fetch_official_index_async` works asynchronously.
    ///
    /// Inputs:
    /// - Async function call.
    ///
    /// Output:
    /// - Future that resolves to `Result<OfficialIndex>`.
    ///
    /// Details:
    /// - Tests that async version works correctly.
    async fn fetch_official_index_async_works() {
        let result = fetch_official_index_async().await;
        // Result depends on system state
        // We just verify it returns a Result and doesn't panic
        if let Ok(index) = result {
            // Success
            assert!(index.pkgs.is_empty() || !index.pkgs.is_empty());
        } else {
            // Both methods failed, which is acceptable in test environment
        }
    }
}