o2_tools/market_data/

market_data.rs

use crate::{
    market_data::order_book::{
        Book,
        BookConfig,
    },
    order_book::Side,
};
use anyhow::{
    Result,
    anyhow,
};
use fuels::types::{
    AssetId,
    Identity,
};
use std::{
    fs::{
        self,
        File,
    },
    io::Read,
    path::PathBuf,
};

#[derive(Debug, Clone)]
pub struct OrderData {
    pub price: u64,
    pub quantity: u64,
    pub side: Side,
    pub trader_id: Identity,
}

pub struct MarketData {
    cache_dir: PathBuf,
}

#[derive(Debug, Clone)]
pub enum OrderBookAction {
    CreateOrder(OrderData),
    CancelOrder(u64),
}

pub mod order_book;

impl MarketData {
    pub fn new() -> Result<Self> {
        let cache_dir = std::env::temp_dir().join("binance_market_data_cache");
        fs::create_dir_all(&cache_dir)?;
        Ok(Self { cache_dir })
    }

    pub async fn download_market_data(&self, pair: &str, date: &str) -> Result<PathBuf> {
        let csv_path = self.cache_dir.join(format!("{pair}-trades-{date}.csv"));

        if csv_path.exists() {
            println!("Data already downloaded");
            return Ok(csv_path);
        }

        let url = format!(
            "https://data.binance.vision/data/spot/daily/trades/{pair}/{pair}-trades-{date}.zip"
        );
        let response = reqwest::get(&url).await?;
        if !response.status().is_success() {
            return Err(anyhow!("Failed to download: {}", response.status()));
        }

        let bytes = response.bytes().await?;
        let reader = std::io::Cursor::new(bytes);
        let mut archive = zip::ZipArchive::new(reader)?;

        if archive.len() != 1 {
            return Err(anyhow!("Expected exactly one file in the archive"));
        }

        let mut zip_file = archive.by_index(0)?;
        let mut csv_data = Vec::new();
        zip_file.read_to_end(&mut csv_data)?;

        fs::write(&csv_path, csv_data)?;
        Ok(csv_path)
    }

    pub fn get_order_data(
        &self,
        pair: &str,
        date: &str,
        limit: usize,
    ) -> Result<Vec<OrderData>> {
        let csv_path = self.cache_dir.join(format!("{pair}-trades-{date}.csv"));

        if !csv_path.exists() {
            return Err(anyhow!(
                "Data not downloaded. Call download_market_data first."
            ));
        }

        let file = File::open(csv_path)?;
        let mut rdr = csv::Reader::from_reader(file);

        let mut orders = Vec::new();
        for (idx, result) in rdr.records().enumerate() {
            if idx >= limit {
                break;
            }

            let record = result?;

            let price: f64 = record
                .get(1)
                .ok_or_else(|| anyhow!("Missing price field"))?
                .parse()?;
            let quantity: f64 = record
                .get(2)
                .ok_or_else(|| anyhow!("Missing quantity field"))?
                .parse()?;
            let is_buyer_maker_str = record
                .get(5)
                .ok_or_else(|| anyhow!("Missing is_buyer_maker field"))?;
            let is_buyer_maker = is_buyer_maker_str.to_lowercase() == "true";
            let price_u64 = (price * 1_000_000.0) as u64;
            let quantity_u64 = (quantity * 1_000_000_000.0) as u64;

            orders.push(OrderData {
                trader_id: Identity::default(),
                price: price_u64,
                quantity: quantity_u64,
                side: if is_buyer_maker {
                    Side::Sell
                } else {
                    Side::Buy
                },
            });
        }
        Ok(orders)
    }

    pub async fn get_ethusdc_orders_data(
        &self,
        date: &str,
        limit: usize,
    ) -> Result<Vec<OrderData>> {
        let pair = "ETHUSDC";
        self.download_market_data(pair, date).await?;
        self.get_order_data(pair, date, limit)
    }

    /// Downloads and retrieves summarized ETHUSDC order data by grouping trades.
    ///
    /// This method reduces the number of orders by grouping consecutive trades and calculating
    /// statistical aggregates for each group. This is useful when you need fewer data points
    /// while preserving the statistical characteristics of real market data.
    ///
    /// # How it works
    ///
    /// 1. Downloads all trade data for the specified date
    /// 2. Calculates group size: `total_trades / limit` (rounded up)
    /// 3. Groups consecutive trades into chunks of the calculated size
    /// 4. For each group, creates a single summarized order with:
    ///    - **Price**: Average price of all trades in the group
    ///    - **Quantity**: Sum of all quantities in the group
    ///    - **Side**: Majority side (Buy/Sell) based on trade count
    ///
    /// # Parameters
    ///
    /// * `date` - Optional date string in "YYYY-MM-DD" format. Defaults to "2025-07-15"
    /// * `limit` - Maximum number of summarized orders to return.
    ///
    /// # Returns
    ///
    /// Returns a `Result<Vec<OrderData>>` containing up to `limit` summarized orders.
    /// Each order represents aggregated data from multiple real trades.
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - Network download fails
    /// - File I/O operations fail
    /// - CSV parsing fails
    /// - Invalid date format
    pub async fn get_ethusdc_orders_data_summrized(
        &self,
        date: &str,
        limit: usize,
    ) -> Result<Vec<OrderBookAction>> {
        let pair = "ETHUSDC";
        self.download_market_data(pair, date).await?;

        // Get all order data first (without limit)
        let all_data = self.get_order_data(pair, date, usize::MAX)?;

        if all_data.is_empty() || limit == 0 {
            return Ok(Vec::new());
        }

        let group_size = all_data.len() / limit;
        let mut summarized_orders = Vec::new();

        // Group trades and calculate averages
        for chunk in all_data.chunks_exact(group_size) {
            if chunk.is_empty() {
                continue;
            }

            // Calculate average price
            let total_price = chunk.iter().map(|order| order.price).sum::<u64>();
            let total_quantity = chunk.iter().map(|order| order.quantity).sum::<u64>();
            let avg_price = total_price / chunk.len() as u64;
            let avg_quantity = total_quantity / chunk.len() as u64;

            // Sum quantities
            let buy_count = chunk
                .iter()
                .filter(|order| matches!(order.side, Side::Buy))
                .count();
            let sell_count = chunk.len() - buy_count;
            let majority_side = if buy_count > sell_count {
                Side::Buy
            } else {
                Side::Sell
            };

            let order = OrderBookAction::CreateOrder(OrderData {
                trader_id: Identity::default(),
                price: avg_price / 1_000_000,
                quantity: avg_quantity / 1_000_000,
                side: majority_side,
            });
            summarized_orders.push(order);
        }

        Ok(summarized_orders)
    }

    pub fn generate_orderbook_state(
        base_asset: AssetId,
        quote_asset: AssetId,
        actions: &[OrderBookAction],
    ) -> Book {
        let mut order_book = Book::new(BookConfig {
            base_asset,
            base_decimals: 9,
            quote_asset,
            quote_decimals: 6,
            taker_fee: 0,
            maker_fee: 0,
        });

        for action in actions {
            match action {
                OrderBookAction::CreateOrder(order) => {
                    order_book.create_order(order.clone());
                }
                OrderBookAction::CancelOrder(order_id) => {
                    order_book.cancel(*order_id);
                }
            }
        }

        order_book
    }
}

// #[cfg(test)]
// mod tests {
//     use super::*;
//     use tokio;

//     #[tokio::test]
//     async fn test_download_and_cache() -> Result<()> {
//         let manager = MarketData::new()?;
//         let date = "2025-07-15";
//         let pair = "ETHUSDC";

//         let csv_path = manager.download_market_data(pair, date).await?;
//         assert!(csv_path.exists());

//         let file_metadata1 = fs::metadata(&csv_path)?;
//         let modified1 = file_metadata1.modified()?;

//         let csv_path2 = manager.download_market_data(pair, date).await?;
//         assert_eq!(csv_path, csv_path2);

//         let file_metadata2 = fs::metadata(&csv_path2)?;
//         let modified2 = file_metadata2.modified()?;
//         assert_eq!(modified1, modified2);

//         Ok(())
//     }

//     #[tokio::test]
//     async fn get_ethusdc_orders_data() -> Result<()> {
//         let manager = MarketData::new()?;
//         let date = "2025-07-15";
//         let orders = manager.get_ethusdc_orders_data(date, 10).await?;
//         assert_eq!(orders.len(), 10);

//         for order in &orders {
//             assert!(order.price > 0);
//             assert!(order.quantity > 0);
//             assert!(matches!(order.side, Side::Buy | Side::Sell));
//         }

//         Ok(())
//     }

//     #[tokio::test]
//     async fn test_get_ethusdc_orders_data_summrized() -> Result<()> {
//         let manager = MarketData::new()?;
//         let date = "2025-07-15";

//         // Test with a limit that should result in grouping
//         let summarized_orders =
//             manager.get_ethusdc_orders_data_summrized(date, 5).await?;
//         println!("summarized_orders: {:?}", summarized_orders.len());
//         assert!(summarized_orders.len() <= 5);

//         // Verify all orders have valid data
//         for order in &summarized_orders {
//             match order {
//                 OrderBookAction::CreateOrder(order) => {
//                     assert!(order.price > 0);
//                     assert!(order.quantity > 0);
//                     assert!(matches!(order.side, Side::Buy | Side::Sell));
//                 }
//                 OrderBookAction::CancelOrder(order_id) => {
//                     assert!(*order_id > 0);
//                 }
//             }
//         }
//         assert_eq!(summarized_orders.len(), 5);

//         Ok(())
//     }
// }