bgustscraper 0.2.0

use anyhow::Result;
use scraper::{Html, Selector};
use url::Url;

pub struct ExtractedDocument {
    pub url: String,
    pub extension: String,
    pub name: String,
}

pub fn find_documents(html: &str, base_url: &str) -> Vec<ExtractedDocument> {
    let document = Html::parse_document(html);
    let selector = Selector::parse("a").unwrap();
    let base = Url::parse(base_url).unwrap();

    let mut docs = Vec::new();
    let target_extensions = ["pdf", "csv", "xlsx", "xls", "ods"];

    for element in document.select(&selector) {
        if let Some(href) = element.value().attr("href") {
            if let Ok(full_url) = base.join(href) {
                let url_str = full_url.to_string();
                let lower_url = url_str.to_lowercase();
                
                for ext in target_extensions {
                    if lower_url.ends_with(&format!(".{}", ext)) {
                        docs.push(ExtractedDocument {
                            url: url_str.clone(),
                            extension: ext.to_string(),
                            name: url_str.split('/').last().unwrap_or("unknown").to_string(),
                        });
                        break;
                    }
                }
            }
        }
    }

    docs
}