yf-common

Shared infrastructure library for Yahoo Finance data extraction tools.

Overview

yf-common provides battle-tested components for building reliable Yahoo Finance data extraction tools:

• HTTP Client - Builder pattern with configurable timeouts and connection pooling
• Rate Limiting - Token bucket algorithm to respect API limits (default: 5 req/min)
• Retry Logic - Exponential backoff with jitter for transient failures
• Authentication - Cookie/crumb mechanism for Yahoo Finance API
• Timestamp Utilities - Parse dates, convert to Unix timestamps
• Output Writers - JSON and CSV formatters

Installation

Add to your Cargo.toml:

[dependencies]
yf-common = "0.2"

# With authentication support (required for options data)
yf-common = { version = "0.2", features = ["auth"] }

# With CSV output support
yf-common = { version = "0.2", features = ["csv"] }

# All features
yf-common = { version = "0.2", features = ["auth", "csv"] }

Features

Quick Start

Basic HTTP Client

use yf_common::{YahooClientBuilder, RetryConfig};
use std::time::Duration;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = YahooClientBuilder::new()
        .with_rate_limit(10)  // 10 requests per minute
        .with_timeout(Duration::from_secs(30))
        .with_retry(RetryConfig::default())
        .build()?;

    let response = client.get("https://query1.finance.yahoo.com/v8/finance/chart/AAPL")
        .await?;

    println!("{}", response);
    Ok(())
}

With Authentication (for Options Data)

use yf_common::{YahooClientBuilder, CrumbAuth};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let auth = CrumbAuth::new();

    let client = YahooClientBuilder::new()
        .with_rate_limit(5)
        .with_crumb_auth(auth)
        .build()?;

    // Authenticate (fetches cookies and crumb)
    client.authenticate().await?;

    // Now make authenticated requests
    let response = client.get("https://query1.finance.yahoo.com/v7/finance/options/AAPL")
        .await?;

    Ok(())
}

Timestamp Utilities

use yf_common::{parse_date_to_timestamp, timestamp_to_date, today_str};

// Parse date string to Unix timestamp
let ts = parse_date_to_timestamp("2024-01-15")?;
assert_eq!(ts, 1705363199);  // End of day UTC

// Convert timestamp back to date string
let date = timestamp_to_date(1705363199);
assert_eq!(date, "2024-01-15");

// Get today's date
let today = today_str();  // e.g., "2024-03-15"

Output Writers

use yf_common::JsonWriter;
use serde::Serialize;

#[derive(Serialize)]
struct StockData {
    symbol: String,
    price: f64,
}

let data = StockData { symbol: "AAPL".into(), price: 185.50 };

// Pretty JSON to stdout
JsonWriter::stdout().pretty().write(&data)?;

// Compact JSON to file
JsonWriter::file("output.json")?.write(&data)?;

Module Reference

error - Error Types

use yf_common::{YfCommonError, Result};

// Error variants:
// - HttpError(reqwest::Error)
// - RateLimitExceeded
// - AuthenticationFailed(String)
// - InvalidResponse(String)
// - ParseError(String)
// - IoError(std::io::Error)

rate_limit - Rate Limiting

use yf_common::{RateLimitConfig, YfRateLimiter};

// Configure rate limiter
let config = RateLimitConfig::new(10);  // 10 requests per minute
let limiter = YfRateLimiter::new(config);

// Check before making request
limiter.acquire().await;  // Blocks if rate exceeded

retry - Retry Logic

use yf_common::{RetryConfig, BackoffStrategy, retry_with_backoff};

let config = RetryConfig {
    max_attempts: 3,
    initial_delay_ms: 1000,
    max_delay_ms: 30000,
    strategy: BackoffStrategy::ExponentialWithJitter,
};

// Retry a fallible async operation
let result = retry_with_backoff(&config, || async {
    // Your async operation here
    Ok::<_, YfCommonError>("success")
}).await?;

client - HTTP Client

use yf_common::{YahooClient, YahooClientBuilder};

let client = YahooClientBuilder::new()
    .with_rate_limit(5)           // Requests per minute
    .with_timeout(Duration::from_secs(30))
    .with_retry(RetryConfig::default())
    .with_user_agent("MyApp/1.0")
    .build()?;

// GET request
let body = client.get("https://example.com/api").await?;

// GET with query params
let body = client.get_with_params(
    "https://example.com/api",
    &[("symbol", "AAPL"), ("range", "1y")]
).await?;

auth - Authentication (feature: auth)

use yf_common::{AuthProvider, CrumbAuth, NoAuth};

// For endpoints requiring authentication
let auth = CrumbAuth::new();
auth.authenticate(&http_client).await?;
let crumb = auth.get_crumb()?;

// For public endpoints
let no_auth = NoAuth;

time - Timestamp Utilities

use yf_common::{
    parse_date_to_timestamp,      // "2024-01-15" -> Unix timestamp (end of day)
    parse_start_date_to_timestamp, // "2024-01-15" -> Unix timestamp (start of day)
    timestamp_to_date,            // Unix timestamp -> "2024-01-15"
    today_utc,                    // Current date as NaiveDate
    today_str,                    // Current date as "YYYY-MM-DD"
};

output - Output Writers

use yf_common::{JsonWriter, CsvWriter};  // CsvWriter requires "csv" feature

// JSON output
JsonWriter::stdout().pretty().write(&data)?;
JsonWriter::file("out.json")?.write(&data)?;

// CSV output (with csv feature)
CsvWriter::stdout().write_records(&records)?;
CsvWriter::file("out.csv")?.with_headers(&["col1", "col2"]).write_records(&records)?;

Architecture

yf-common/
├── Cargo.toml
├── README.md
└── src/
    ├── lib.rs          # Public API exports
    ├── error.rs        # YfCommonError enum
    ├── client.rs       # YahooClient with builder pattern
    ├── auth.rs         # CrumbAuth (feature-gated)
    ├── retry.rs        # Exponential backoff
    ├── rate_limit.rs   # Governor-based rate limiting
    ├── time.rs         # Timestamp utilities
    └── output.rs       # JSON/CSV writers

Used By

• yf-quotes - Historical OHLCV stock data
• yf-options - Options chain with Greeks

MSRV

Minimum Supported Rust Version: 1.75

License

Licensed under either of:

• Apache License, Version 2.0 (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0)
• MIT license (LICENSE-MIT or http://opensource.org/licenses/MIT)

at your option.

Contributing

Contributions welcome! Please read the contributing guidelines first.