bgustscraper 0.2.0

use anyhow::{Result, anyhow};
use std::process::{Command, Stdio, Child};
use std::io::{Write, BufReader, BufRead};
use serde::{Serialize, Deserialize};

#[derive(Serialize)]
struct ScrapeRequest {
    action: String,
    url: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    options: Option<ScrapeOptions>,
    #[serde(skip_serializing_if = "Option::is_none")]
    proxy: Option<String>,
}

#[derive(Serialize)]
pub struct ScrapeOptions {
    pub deep_scan: bool,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub semantic_embeddings: Option<bool>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub text: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prompt: Option<String>,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ScrapeResponse {
    pub status: String,
    pub html: Option<String>,
    pub markdown: Option<String>,
    pub title: Option<String>,
    pub url: Option<String>,
    pub message: Option<String>,
    pub sentences: Option<Vec<String>>,
    pub embeddings: Option<Vec<Vec<f32>>>,
    pub data: Option<serde_json::Value>,
}

#[derive(Clone, Copy)]
pub enum ScrapingEngine {
    Playwright,
    Cloudscraper,
}

pub struct PlaywrightWorker {
    child: Child,
    pub engine: ScrapingEngine,
    proxies: Vec<String>,
    current_proxy_idx: usize,
}

impl PlaywrightWorker {
    pub fn new(headless: bool, engine: ScrapingEngine) -> Result<Self> {
        let script = match engine {
            ScrapingEngine::Playwright => "worker/index.js",
            ScrapingEngine::Cloudscraper => "worker/cloudscraper_worker.js",
        };

        let child = Command::new("node")
            .arg(script)
            .arg(if headless { "true" } else { "false" })
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::inherit())
            .spawn()?;

        // Técnica A: Proxy List (Podríamos cargar esto de un archivo externo)
        let proxies = vec![
            // "http://username:password@proxy-server:port",
        ];

        Ok(Self { 
            child, 
            engine, 
            proxies,
            current_proxy_idx: 0 
        })
    }

    pub fn scrape(&mut self, url: &str, options: Option<ScrapeOptions>) -> Result<ScrapeResponse> {
        let proxy = if self.proxies.is_empty() {
            None
        } else {
            let p = Some(self.proxies[self.current_proxy_idx].clone());
            self.current_proxy_idx = (self.current_proxy_idx + 1) % self.proxies.len();
            p
        };

        let stdin = self.child.stdin.as_mut().expect("Failed to open stdin");
        let action = if url == "extract" {
            "extract".to_string()
        } else {
            "scrape".to_string()
        };

        let request = ScrapeRequest {
            action,
            url: url.to_string(),
            options,
            proxy,
        };

        let json = serde_json::to_string(&request)?;
        writeln!(stdin, "{}", json)?;

        let stdout = self.child.stdout.as_mut().expect("Failed to open stdout");
        let mut reader = BufReader::new(stdout);
        let mut line = String::new();
        reader.read_line(&mut line)?;

        if line.trim().is_empty() {
            return Err(anyhow!("Worker returned empty response. Possibly crashed."));
        }

        let response: ScrapeResponse = serde_json::from_str(&line)?;
        Ok(response)
    }

    pub fn exit(&mut self) -> Result<()> {
        let stdin = self.child.stdin.as_mut().expect("Failed to open stdin");
        let request = ScrapeRequest {
            action: "exit".to_string(),
            url: "".to_string(),
            options: None,
            proxy: None,
        };
        let json = serde_json::to_string(&request)?;
        writeln!(stdin, "{}", json)?;
        self.child.wait()?;
        Ok(())
    }
}