rust_scrapling 0.4.1

use clap::{Parser, Subcommand, ValueEnum};
use rust_scrapling::fetchers::client::Fetcher;
use rust_scrapling::fetchers::config::FetcherConfig;

/// Output format for the `fetch` subcommand. A ValueEnum so clap validates
/// the value and lists the choices in --help, instead of silently treating
/// any typo as "text".
#[derive(Clone, Copy, PartialEq, Eq, ValueEnum)]
enum OutputFormat {
    /// Extracted text content
    Text,
    /// Raw HTML
    Html,
    /// JSON objects with tag/text/html per element
    Json,
}

#[derive(Parser)]
#[command(name = "rust-scrapling")]
#[command(about = "RUSTScrapling - A Rust web scraping framework", version)]
struct Cli {
    #[command(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    /// Fetch a URL and extract content
    Fetch {
        url: String,
        #[arg(short, long)]
        selector: Option<String>,
        #[arg(short, long, value_enum, default_value_t = OutputFormat::Text)]
        format: OutputFormat,
        #[arg(long)]
        no_stealth: bool,
    },
    /// Extract text content from a URL
    Extract {
        url: String,
        #[arg(short, long)]
        selector: Option<String>,
    },
}

#[tokio::main]
async fn main() {
    env_logger::init();
    let cli = Cli::parse();

    match cli.command {
        Commands::Fetch {
            url,
            selector,
            format,
            no_stealth,
        } => {
            let config = FetcherConfig::builder().stealth(!no_stealth).build();
            let fetcher = match Fetcher::new(config) {
                Ok(f) => f,
                Err(e) => {
                    eprintln!("Failed to build HTTP client: {}", e);
                    std::process::exit(1);
                }
            };
            match fetcher.get(&url).await {
                Ok(response) => {
                    if let Some(css) = selector {
                        let sel = response.selector();
                        let results = sel.css(&css);
                        for item in &results {
                            match format {
                                OutputFormat::Html => println!("{}", item.outer_html()),
                                OutputFormat::Json => {
                                    let obj = serde_json::json!({
                                        "tag": item.tag(),
                                        "text": item.text().as_str(),
                                        "html": item.html_content().as_str(),
                                    });
                                    println!("{}", serde_json::to_string_pretty(&obj).unwrap());
                                }
                                OutputFormat::Text => println!("{}", item.text()),
                            }
                        }
                        eprintln!("Found {} elements", results.len());
                    } else {
                        match format {
                            OutputFormat::Html => println!("{}", response.text()),
                            OutputFormat::Text | OutputFormat::Json => {
                                let sel = response.selector();
                                println!(
                                    "{}",
                                    sel.get_all_text("\n", true, &["script", "style"], None)
                                );
                            }
                        }
                    }
                }
                Err(e) => {
                    eprintln!("Error: {}", e);
                    std::process::exit(1);
                }
            }
        }
        Commands::Extract { url, selector } => {
            let fetcher = match Fetcher::new(FetcherConfig::default()) {
                Ok(f) => f,
                Err(e) => {
                    eprintln!("Failed to build HTTP client: {}", e);
                    std::process::exit(1);
                }
            };
            match fetcher.get(&url).await {
                Ok(response) => {
                    let sel = response.selector();
                    if let Some(css) = selector {
                        let results = sel.css(&css);
                        for item in &results {
                            println!("{}", item.text());
                        }
                    } else {
                        println!(
                            "{}",
                            sel.get_all_text("\n", true, &["script", "style"], None)
                        );
                    }
                }
                Err(e) => {
                    eprintln!("Error: {}", e);
                    std::process::exit(1);
                }
            }
        }
    }
}