1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
use clap::{Parser, Subcommand};
use rust_scrapling::fetchers::client::Fetcher;
use rust_scrapling::fetchers::config::FetcherConfig;
#[derive(Parser)]
#[command(name = "rust-scrapling")]
#[command(about = "RUSTScrapling - A Rust web scraping framework", version)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Fetch a URL and extract content
Fetch {
url: String,
#[arg(short, long)]
selector: Option<String>,
#[arg(short, long, default_value = "text")]
format: String,
#[arg(long)]
no_stealth: bool,
},
/// Extract text content from a URL
Extract {
url: String,
#[arg(short, long)]
selector: Option<String>,
},
}
#[tokio::main]
async fn main() {
env_logger::init();
let cli = Cli::parse();
match cli.command {
Commands::Fetch {
url,
selector,
format,
no_stealth,
} => {
let config = FetcherConfig::builder().stealth(!no_stealth).build();
let fetcher = match Fetcher::new(config) {
Ok(f) => f,
Err(e) => {
eprintln!("Failed to build HTTP client: {}", e);
std::process::exit(1);
}
};
match fetcher.get(&url).await {
Ok(response) => {
if let Some(css) = selector {
let sel = response.selector();
let results = sel.css(&css);
for item in &results {
match format.as_str() {
"html" => println!("{}", item.outer_html()),
"json" => {
let obj = serde_json::json!({
"tag": item.tag(),
"text": item.text().as_str(),
"html": item.html_content().as_str(),
});
println!("{}", serde_json::to_string_pretty(&obj).unwrap());
}
_ => println!("{}", item.text()),
}
}
eprintln!("Found {} elements", results.len());
} else {
match format.as_str() {
"html" => println!("{}", response.text()),
_ => {
let sel = response.selector();
println!(
"{}",
sel.get_all_text("\n", true, &["script", "style"], None)
);
}
}
}
}
Err(e) => {
eprintln!("Error: {}", e);
std::process::exit(1);
}
}
}
Commands::Extract { url, selector } => {
let fetcher = match Fetcher::new(FetcherConfig::default()) {
Ok(f) => f,
Err(e) => {
eprintln!("Failed to build HTTP client: {}", e);
std::process::exit(1);
}
};
match fetcher.get(&url).await {
Ok(response) => {
let sel = response.selector();
if let Some(css) = selector {
let results = sel.css(&css);
for item in &results {
println!("{}", item.text());
}
} else {
println!(
"{}",
sel.get_all_text("\n", true, &["script", "style"], None)
);
}
}
Err(e) => {
eprintln!("Error: {}", e);
std::process::exit(1);
}
}
}
}
}