finance_query/adapters/edgar/

mod.rs

//! SEC EDGAR API client.
//!
//! Provides access to SEC EDGAR data including filing history,
//! structured XBRL financial data, and full-text search.
//!
//! All requests are rate-limited to 10 per second as required by SEC.
//! Rate limiting and CIK caching are managed via a process-global singleton.
//!
//! # Quick Start
//!
//! Initialize once at application startup, then use anywhere:
//!
//! ```no_run
//! use finance_query::edgar;
//!
//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
//! // Initialize once (required)
//! edgar::init("user@example.com")?;
//!
//! // Use anywhere
//! let cik = edgar::resolve_cik("AAPL").await?;
//! let submissions = edgar::submissions(cik).await?;
//! let facts = edgar::company_facts(cik).await?;
//!
//! // Search filings
//! let results = edgar::search(
//!     "artificial intelligence",
//!     Some(&["10-K"]),
//!     Some("2024-01-01"),
//!     None,
//!     None,
//!     None,
//! ).await?;
//! # Ok(())
//! # }
//! ```

mod client;
pub(crate) mod discovery; // DISCOVERY
mod endpoints;
pub(crate) mod filings; // FILINGS

use crate::error::{FinanceError, Result};
use crate::models::filings::{
    CompanyFacts, EdgarFilingIndex, EdgarSearchResults, EdgarSubmissions,
};
use crate::rate_limiter::RateLimiter;
use client::EdgarClientBuilder;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::sync::RwLock;

/// SEC EDGAR rate limit: 10 requests per second.
const EDGAR_RATE_PER_SEC: f64 = 10.0;

/// Stable configuration stored in the EDGAR process-global singleton.
///
/// Only configuration, the rate limiter, and the CIK cache are stored — NOT
/// the `reqwest::Client`. `reqwest::Client` internally spawns hyper
/// connection-pool tasks on whichever tokio runtime first uses them; when that
/// runtime is dropped (e.g. at the end of a `#[tokio::test]`), those tasks die
/// and subsequent calls from a different runtime receive `DispatchGone`. A fresh
/// `reqwest::Client` is built per public function call via
/// [`EdgarClientBuilder::build_with_shared_state`], reusing the shared rate
/// limiter and CIK cache.
struct EdgarSingleton {
    email: String,
    app_name: String,
    timeout: Duration,
    rate_limiter: Arc<RateLimiter>,
    cik_cache: Arc<RwLock<Option<HashMap<String, u64>>>>,
}

static EDGAR_SINGLETON: OnceLock<EdgarSingleton> = OnceLock::new();

/// Initialize the global EDGAR client with a contact email.
///
/// This function must be called once before using any EDGAR functions.
/// The SEC requires all automated requests to include a User-Agent header
/// with a contact email address.
///
/// # Arguments
///
/// * `email` - Contact email address (included in User-Agent header)
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// # Ok(())
/// # }
/// ```
///
/// # Errors
///
/// Returns an error if EDGAR has already been initialized.
pub fn init(email: impl Into<String>) -> Result<()> {
    init_with_config(email, "finance-query", Duration::from_secs(30))
}

/// Initialize the global EDGAR client with full configuration.
///
/// Use this for custom app name and timeout settings.
///
/// # Arguments
///
/// * `email` - Contact email address (required by SEC)
/// * `app_name` - Application name (included in User-Agent)
/// * `timeout` - HTTP request timeout duration
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
/// use std::time::Duration;
///
/// # fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init_with_config(
///     "user@example.com",
///     "my-app",
///     Duration::from_secs(60),
/// )?;
/// # Ok(())
/// # }
/// ```
pub fn init_with_config(
    email: impl Into<String>,
    app_name: impl Into<String>,
    timeout: Duration,
) -> Result<()> {
    init_with_rate(email, app_name, timeout, EDGAR_RATE_PER_SEC)
}

/// Initialize the global EDGAR client with a request rate below SEC's limit.
///
/// Long bulk downloads may prefer a margin under the 10 requests a second SEC
/// allows. Rates above that limit are capped to it.
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
/// use std::time::Duration;
///
/// # fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init_with_rate("user@example.com", "my-app", Duration::from_secs(60), 8.0)?;
/// # Ok(())
/// # }
/// ```
///
/// # Errors
///
/// Returns an error if EDGAR has already been initialized or the rate isn't positive.
pub fn init_with_rate(
    email: impl Into<String>,
    app_name: impl Into<String>,
    timeout: Duration,
    requests_per_second: f64,
) -> Result<()> {
    if requests_per_second.is_nan() || requests_per_second <= 0.0 {
        return Err(FinanceError::InvalidParameter {
            param: "requests_per_second".to_string(),
            reason: "must be positive".to_string(),
        });
    }
    EDGAR_SINGLETON
        .set(EdgarSingleton {
            email: email.into(),
            app_name: app_name.into(),
            timeout,
            rate_limiter: Arc::new(RateLimiter::new(
                requests_per_second.min(EDGAR_RATE_PER_SEC),
            )),
            cik_cache: Arc::new(RwLock::new(None)),
        })
        .map_err(|_| FinanceError::InvalidParameter {
            param: "edgar".to_string(),
            reason: "EDGAR client already initialized".to_string(),
        })
}

/// Build a fresh [`EdgarClient`](client::EdgarClient) from the singleton's
/// config, reusing the shared rate limiter and CIK cache.
///
/// If EDGAR hasn't been explicitly initialized, falls back to the `EDGAR_EMAIL`
/// environment variable as a convenience (consistent with other adapters).
fn build_client() -> Result<client::EdgarClient> {
    if EDGAR_SINGLETON.get().is_none()
        && let Ok(email) = std::env::var("EDGAR_EMAIL")
    {
        let _ = EDGAR_SINGLETON.set(EdgarSingleton {
            email,
            app_name: "finance-query".to_string(),
            timeout: Duration::from_secs(30),
            rate_limiter: Arc::new(RateLimiter::new(EDGAR_RATE_PER_SEC)),
            cik_cache: Arc::new(RwLock::new(None)),
        });
    }
    let s = EDGAR_SINGLETON
        .get()
        .ok_or_else(|| FinanceError::InvalidParameter {
            param: "edgar".to_string(),
            reason: "EDGAR_EMAIL not set. Call edgar::init(email) or set EDGAR_EMAIL env var."
                .to_string(),
        })?;
    EdgarClientBuilder::new(&s.email)
        .app_name(&s.app_name)
        .timeout(s.timeout)
        .build_with_shared_state(Arc::clone(&s.rate_limiter), Arc::clone(&s.cik_cache))
}

fn accession_parts(accession_number: &str) -> Result<(String, String)> {
    let cik_part = accession_number
        .split('-')
        .next()
        .unwrap_or("")
        .trim_start_matches('0')
        .to_string();
    let accession_no_dashes = accession_number.replace('-', "");

    if cik_part.is_empty() || accession_no_dashes.is_empty() {
        return Err(FinanceError::InvalidParameter {
            param: "accession_number".to_string(),
            reason: "Invalid accession number format".to_string(),
        });
    }

    Ok((cik_part, accession_no_dashes))
}

/// Resolve a ticker symbol to its SEC CIK number.
///
/// The ticker-to-CIK mapping is fetched once and cached process-wide.
/// Lookups are case-insensitive.
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// let cik = edgar::resolve_cik("AAPL").await?;
/// assert_eq!(cik, 320193);
/// # Ok(())
/// # }
/// ```
///
/// # Errors
///
/// Returns an error if:
/// - EDGAR has not been initialized (call `init()` first)
/// - Symbol not found in SEC database
/// - Network request fails
pub async fn resolve_cik(symbol: &str) -> Result<u64> {
    build_client()?.resolve_cik(symbol).await
}

/// Fetch filing history and company metadata for a CIK.
///
/// Returns the most recent ~1000 filings inline, with references to
/// additional history files for older filings.
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// let cik = edgar::resolve_cik("AAPL").await?;
/// let submissions = edgar::submissions(cik).await?;
/// println!("Company: {:?}", submissions.name);
/// # Ok(())
/// # }
/// ```
pub async fn submissions(cik: u64) -> Result<EdgarSubmissions> {
    build_client()?.submissions(cik).await
}

/// Fetch structured XBRL financial data for a CIK.
///
/// Returns all extracted XBRL facts organized by taxonomy (us-gaap, ifrs, dei).
/// This can be a large response (several MB for major companies).
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// let cik = edgar::resolve_cik("AAPL").await?;
/// let facts = edgar::company_facts(cik).await?;
/// println!("Entity: {:?}", facts.entity_name);
/// # Ok(())
/// # }
/// ```
pub async fn company_facts(cik: u64) -> Result<CompanyFacts> {
    build_client()?.company_facts(cik).await
}

/// Fetch any EDGAR file as bytes, within the shared rate limit.
///
/// Covers what the typed functions don't: filing documents and archives under
/// `https://www.sec.gov/Archives/` (index pages, XBRL zips, instance documents,
/// primary documents, the quarterly and daily indexes) and the JSON under
/// `https://data.sec.gov/` (including older submissions pages), and the standard
/// taxonomy schemas filings import (`xbrl.sec.gov`, `xbrl.fasb.org`,
/// `taxonomies.xbrl.us`). Other hosts are refused, so the SEC User-Agent and
/// rate limit aren't spent elsewhere.
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// let index = edgar::archive("https://www.sec.gov/Archives/edgar/full-index/2025/QTR1/xbrl.idx").await?;
/// println!("{} bytes", index.len());
/// # Ok(())
/// # }
/// ```
///
/// # Errors
///
/// Returns an error for a URL outside those locations, a missing file
/// (`SymbolNotFound`), rate limiting (`RateLimited`) or a server error.
pub async fn archive(url: &str) -> Result<Vec<u8>> {
    if !is_archive_url(url) {
        return Err(FinanceError::InvalidParameter {
            param: "url".to_string(),
            reason: "must be an EDGAR archive, data.sec.gov or standard taxonomy URL".to_string(),
        });
    }
    build_client()?.get_document(url).await
}

const ARCHIVE_PREFIXES: &[&str] = &[
    "https://www.sec.gov/Archives/",
    "https://data.sec.gov/",
    "https://xbrl.sec.gov/",
    "https://xbrl.fasb.org/",
    "https://taxonomies.xbrl.us/",
];

/// Checks the parsed form, since the request resolves `..` segments before it is sent.
fn is_archive_url(url: &str) -> bool {
    reqwest::Url::parse(url).is_ok_and(|parsed| {
        ARCHIVE_PREFIXES
            .iter()
            .any(|prefix| parsed.as_str().starts_with(prefix))
    })
}

/// Fetch the filing index for a specific accession number.
///
/// This provides the file list for a filing, which can be used to locate
/// the primary HTML document and file sizes.
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// let index = edgar::filing_index("0000320193-24-000123").await?;
/// println!("Files: {}", index.directory.item.len());
/// # Ok(())
/// # }
/// ```
pub async fn filing_index(accession_number: &str) -> Result<EdgarFilingIndex> {
    build_client()?.filing_index(accession_number).await
}

/// Search SEC EDGAR filings by text content.
///
/// # Arguments
///
/// * `query` - Search term or phrase
/// * `forms` - Optional form type filter (e.g., `&["10-K", "10-Q"]`)
/// * `start_date` - Optional start date (YYYY-MM-DD)
/// * `end_date` - Optional end date (YYYY-MM-DD)
/// * `from` - Optional pagination offset (default: 0)
/// * `size` - Optional page size (default: 100, max: 100)
///
/// # Example
///
/// ```no_run
/// use finance_query::edgar;
///
/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
/// edgar::init("user@example.com")?;
/// let results = edgar::search(
///     "artificial intelligence",
///     Some(&["10-K"]),
///     Some("2024-01-01"),
///     None,
///     Some(0),
///     Some(100),
/// ).await?;
/// if let Some(hits_container) = &results.hits {
///     println!("Found {} results", hits_container.total.as_ref().and_then(|t| t.value).unwrap_or(0));
/// }
/// # Ok(())
/// # }
/// ```
pub async fn search(
    query: &str,
    forms: Option<&[&str]>,
    start_date: Option<&str>,
    end_date: Option<&str>,
    from: Option<usize>,
    size: Option<usize>,
) -> Result<EdgarSearchResults> {
    build_client()?
        .search(query, forms, start_date, end_date, from, size)
        .await
}

// ============================================================================
// Compound operations
// ============================================================================

/// Resolve `symbol` to a CIK and fetch its submissions using a single client.
///
/// Collapses the two HTTP connection pools (and TLS handshakes) that calling
/// [`resolve_cik`] then [`submissions`] would otherwise pay for into one.
pub(crate) async fn submissions_for_symbol(symbol: &str) -> Result<EdgarSubmissions> {
    submissions_for_symbol_with(&build_client()?, symbol).await
}

/// [`submissions_for_symbol`] against a caller-supplied client.
///
/// Callers that make further EDGAR requests off the same submissions list keep
/// one connection pool for the whole sequence instead of one per request.
async fn submissions_for_symbol_with(
    client: &client::EdgarClient,
    symbol: &str,
) -> Result<EdgarSubmissions> {
    let cik = client.resolve_cik(symbol).await?;
    client.submissions(cik).await
}

/// Resolve `symbol` to a CIK and fetch its XBRL company facts using a single client.
pub(crate) async fn company_facts_for_symbol(symbol: &str) -> Result<CompanyFacts> {
    let client = build_client()?;
    let cik = client.resolve_cik(symbol).await?;
    client.company_facts(cik).await
}

// ============================================================================
// Canonical model conversion functions
// ============================================================================

/// Fetch canonical ProviderFilings for a ticker symbol.
pub async fn fetch_filings_response(
    symbol: &str,
) -> Result<crate::models::filings::ProviderFilings> {
    use crate::models::filings::{ProviderFiling, ProviderFilings};

    let subs = submissions_for_symbol(symbol).await?;

    let cik = subs.cik.clone().unwrap_or_default();
    let company_name = subs.name.clone();
    let filings = subs
        .filings
        .and_then(|f| f.recent)
        .map(|r| r.to_filings())
        .unwrap_or_default()
        .into_iter()
        .map(|f| {
            let accession_no_dashes = f.accession_number.replace('-', "");
            let url = if !cik.is_empty()
                && !accession_no_dashes.is_empty()
                && !f.primary_document.is_empty()
            {
                Some(format!(
                    "https://www.sec.gov/Archives/edgar/data/{}/{}/{}",
                    cik.trim_start_matches('0'),
                    accession_no_dashes,
                    f.primary_document
                ))
            } else {
                None
            };
            ProviderFiling {
                accession_number: Some(f.accession_number),
                filing_date: Some(f.filing_date),
                filing_type: Some(f.form),
                filing_url: url,
                company_name: company_name.clone(),
                cik: Some(cik.clone()),
            }
        })
        .collect();

    Ok(ProviderFilings {
        symbol: symbol.to_string(),
        filings,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_init_sets_singleton() {
        let result = init("test@example.com");
        assert!(result.is_ok() || result.is_err()); // May already be initialized
    }

    #[test]
    fn test_double_init_fails() {
        let _ = init("first@example.com");
        let result = init("second@example.com");
        assert!(matches!(result, Err(FinanceError::InvalidParameter { .. })));
    }

    #[test]
    fn test_singleton_is_set_after_init() {
        let _ = init("test@example.com");
        assert!(EDGAR_SINGLETON.get().is_some());
    }

    #[test]
    fn test_init_with_rate_rejects_non_positive_rates() {
        for rate in [0.0, -1.0, f64::NAN] {
            assert!(matches!(
                init_with_rate("test@example.com", "test", Duration::from_secs(1), rate),
                Err(FinanceError::InvalidParameter { .. })
            ));
        }
    }

    #[test]
    fn test_archive_urls_stay_on_sec_hosts() {
        assert!(is_archive_url(
            "https://www.sec.gov/Archives/edgar/full-index/2025/QTR1/xbrl.idx"
        ));
        assert!(is_archive_url(
            "https://data.sec.gov/submissions/CIK0000320193-submissions-001.json"
        ));
        assert!(is_archive_url(
            "https://xbrl.fasb.org/us-gaap/2026/elts/us-gaap-2026.xsd"
        ));
        assert!(is_archive_url("https://xbrl.sec.gov/dei/2026/dei-2026.xsd"));
        assert!(!is_archive_url(
            "http://xbrl.fasb.org/us-gaap/2026/elts/us-gaap-2026.xsd"
        ));
        assert!(!is_archive_url("https://www.sec.gov/cgi-bin/browse-edgar"));
        assert!(!is_archive_url(
            "https://www.sec.gov/Archives/../cgi-bin/browse-edgar"
        ));
        assert!(!is_archive_url(
            "https://www.sec.gov/Archives/%2e%2e/cgi-bin/browse-edgar"
        ));
        assert!(!is_archive_url(
            "http://www.sec.gov/Archives/edgar/data/1/2/x.htm"
        ));
        assert!(!is_archive_url("https://example.com/Archives/"));
    }

    #[tokio::test]
    async fn test_archive_refuses_other_hosts_without_a_request() {
        assert!(matches!(
            archive("https://example.com/file").await,
            Err(FinanceError::InvalidParameter { .. })
        ));
    }
}