1use crate::error::{PopoError, Result};
2use crate::models::{AlertLevel, VolcanoReport, WindDirection};
3use chrono::{Datelike, NaiveDate, Utc};
4use scraper::{Html, Selector};
5
6const BASE_URL: &str = "https://www.cenapred.unam.mx";
7const REPORT_URL: &str = "https://www.cenapred.unam.mx/reportesVolcanesMX/Procesos?tipoProceso=detallesUltimoReporteVolcan";
8
9pub struct Scraper {
10 client: reqwest::blocking::Client,
11}
12
13impl Scraper {
14 pub fn new() -> Self {
15 let client = reqwest::blocking::Client::builder()
16 .user_agent("popo-cli/0.1.0 (+https://github.com/KyleEdwardDonaldson/popo-cli)")
17 .timeout(std::time::Duration::from_secs(30))
18 .build()
19 .expect("Failed to create HTTP client");
20
21 Self { client }
22 }
23
24 pub fn fetch_latest(&self) -> Result<VolcanoReport> {
25 let html = self.client.get(REPORT_URL).send()?.text()?;
26 self.parse_report(&html)
27 }
28
29 pub fn fetch_date(&self, date: NaiveDate) -> Result<VolcanoReport> {
30 let url = format!(
32 "https://www.cenapred.unam.mx/reportesVolcanesMX/Procesos?fecha={}",
33 date.format("%Y-%m-%d")
34 );
35
36 let response = self.client.get(&url).send()?;
37 let html = response.text()?;
38 let report = self.parse_report(&html)?;
39
40 if report.date == date {
42 Ok(report)
43 } else {
44 Err(PopoError::Parse(format!(
45 "Report date mismatch: requested {}, got {}",
46 date, report.date
47 )))
48 }
49 }
50
51 pub fn parse_report(&self, html: &str) -> Result<VolcanoReport> {
52 let document = Html::parse_document(html);
53
54 let date = Self::extract_date(&document)?;
56
57 let exhalations = Self::extract_from_table(&document, "Exhalaciones", &date)?;
59 let volcanotectonic_events = Self::extract_from_table(&document, "Volcanotectónicos", &date)?;
60 let tremor_minutes_total = Self::extract_from_table(&document, "Minutos de tremor", &date)?;
61 let explosions = Self::extract_from_table(&document, "Explosiones", &date)?;
62
63 let (tremor_high_freq, tremor_harmonic) = Self::extract_tremor_breakdown(&document);
65
66 let (alert_level, alert_phase) = Self::extract_alert_info(&document)?;
68
69 let wind_direction = Self::extract_wind_direction(&document);
71
72 let (so2_emissions, so2_date) = Self::extract_so2_data(&document);
74
75 let summary_spanish = Self::extract_summary(&document)?;
77
78 let image_urls = Self::extract_image_urls(&document);
80 let video_urls = Self::extract_video_urls(&document);
81
82 Ok(VolcanoReport {
83 date,
84 report_time: Utc::now(),
85 exhalations,
86 volcanotectonic_events,
87 tremor_minutes_total,
88 tremor_high_frequency_minutes: tremor_high_freq,
89 tremor_harmonic_minutes: tremor_harmonic,
90 explosions,
91 so2_emissions_tons_per_day: so2_emissions,
92 so2_measurement_date: so2_date,
93 alert_level,
94 alert_phase,
95 wind_direction,
96 summary_spanish,
97 image_urls,
98 video_urls,
99 source_url: REPORT_URL.to_string(),
100 scraped_at: Utc::now(),
101 })
102 }
103
104 fn extract_date(document: &Html) -> Result<NaiveDate> {
105 let selector = Selector::parse("h4").unwrap();
106
107 for element in document.select(&selector) {
108 let text = element.text().collect::<String>();
109 let trimmed = text.trim();
110
111 if trimmed.contains("de") && trimmed.contains("de 20") {
113 if let Ok(date) = Self::parse_spanish_date(trimmed) {
114 return Ok(date);
115 }
116 }
117
118 for part in trimmed.split(|c: char| c.is_whitespace() || c == '\u{00A0}') {
121 if part.len() >= 10 {
122 let potential_date = &part[..part.len().min(10)];
123 if potential_date.chars().filter(|&c| c == '-').count() == 2 {
124 if let Ok(date) = NaiveDate::parse_from_str(potential_date, "%Y-%m-%d") {
125 return Ok(date);
126 }
127 }
128 }
129 }
130 }
131
132 Err(PopoError::Parse("Could not find date in report".to_string()))
133 }
134
135 pub fn parse_spanish_date(text: &str) -> Result<NaiveDate> {
136 let parts: Vec<&str> = text.split_whitespace().collect();
137 if parts.len() < 5 {
138 return Err(PopoError::Parse(format!("Invalid date format: {}", text)));
139 }
140
141 let day: u32 = parts[0].parse()
142 .map_err(|_| PopoError::Parse(format!("Invalid day: {}", parts[0])))?;
143
144 let month = match parts[2].to_lowercase().as_str() {
145 "enero" => 1,
146 "febrero" => 2,
147 "marzo" => 3,
148 "abril" => 4,
149 "mayo" => 5,
150 "junio" => 6,
151 "julio" => 7,
152 "agosto" => 8,
153 "septiembre" => 9,
154 "octubre" => 10,
155 "noviembre" => 11,
156 "diciembre" => 12,
157 _ => return Err(PopoError::Parse(format!("Invalid month: {}", parts[2]))),
158 };
159
160 let year: i32 = parts[4].parse()
161 .map_err(|_| PopoError::Parse(format!("Invalid year: {}", parts[4])))?;
162
163 NaiveDate::from_ymd_opt(year, month, day)
164 .ok_or_else(|| PopoError::Parse(format!("Invalid date: {}-{}-{}", year, month, day)))
165 }
166
167 fn extract_from_table(document: &Html, table_name: &str, date: &NaiveDate) -> Result<u32> {
168 let date_str = format!("{:02}-{:02}-{}", date.day(), date.month(), date.year());
169
170 let table_selector = Selector::parse("table").unwrap();
172 let row_selector = Selector::parse("tr").unwrap();
173 let cell_selector = Selector::parse("td").unwrap();
174
175 for table in document.select(&table_selector) {
176 let header_text = table.text().collect::<String>();
177
178 if header_text.contains(table_name) {
179 for row in table.select(&row_selector) {
181 let cells: Vec<_> = row.select(&cell_selector).collect();
182
183 if cells.len() >= 2 {
184 let date_cell = cells[0].text().collect::<String>();
185
186 if date_cell.trim() == date_str {
187 let value_cell = cells[1].text().collect::<String>();
188 let value: u32 = value_cell.trim().parse()
189 .map_err(|_| PopoError::Parse(format!("Invalid value for {}: {}", table_name, value_cell)))?;
190 return Ok(value);
191 }
192 }
193 }
194 }
195 }
196
197 let script_selector = Selector::parse("script").unwrap();
199 for script in document.select(&script_selector) {
200 let script_text = script.text().collect::<String>();
201
202 let column_pattern = format!("addColumn('number', '{}')", table_name);
203 if let Some(column_pos) = script_text.find(&column_pattern) {
204 let after_column = &script_text[column_pos..];
206 if let Some(rows_start) = after_column.find("data.addRows([") {
207 let after_rows_start = &after_column[rows_start + 14..];
208 if let Some(rows_end) = after_rows_start.find("]);") {
209 let data_section = &after_rows_start[..rows_end];
210
211 let target_date_prefix = format!("'{}-{:02}-{:02}", date.year(), date.month(), date.day());
214 for line in data_section.lines() {
215 let trimmed = line.trim();
216 if trimmed.starts_with('[') && trimmed.contains(&target_date_prefix) {
217 let parts: Vec<&str> = trimmed.split(',').collect();
220 if parts.len() >= 2 {
221 let value_str = parts[1].trim();
222 if let Ok(value) = value_str.parse::<u32>() {
223 return Ok(value);
224 }
225 }
226 }
227 }
228 }
229 }
230 }
231 }
232
233 Err(PopoError::Parse(format!("Could not find {} data for date {}", table_name, date_str)))
234 }
235
236 fn extract_tremor_breakdown(document: &Html) -> (Option<u32>, Option<u32>) {
237 let p_selector = Selector::parse("p").unwrap();
238
239 for element in document.select(&p_selector) {
240 let text = element.text().collect::<String>();
241
242 if text.contains("alta frecuencia") && text.contains("armónico") {
244 let high_freq = Self::extract_number_before(&text, "minutos fueron de alta frecuencia");
245 let harmonic = Self::extract_number_before(&text, "minutos de armónico");
246 return (high_freq, harmonic);
247 }
248 }
249
250 (None, None)
251 }
252
253 pub fn extract_number_before(text: &str, marker: &str) -> Option<u32> {
254 if let Some(pos) = text.find(marker) {
255 let before = &text[..pos];
256 let words: Vec<&str> = before.split_whitespace().collect();
257 if let Some(last_word) = words.last() {
258 if let Ok(num) = last_word.parse::<u32>() {
259 return Some(num);
260 }
261 }
262 }
263 None
264 }
265
266 fn extract_alert_info(document: &Html) -> Result<(AlertLevel, String)> {
267 let selector = Selector::parse("p, h4").unwrap();
268
269 for element in document.select(&selector) {
270 let text = element.text().collect::<String>();
271
272 if text.contains("AMARILLO") || text.contains("VERDE") ||
273 text.contains("ROJO") || text.contains("NARANJA") {
274 if let Some(level) = AlertLevel::from_spanish(&text) {
275 let phase = text.trim().to_string();
276 return Ok((level, phase));
277 }
278 }
279 }
280
281 Err(PopoError::Parse("Could not find alert level".to_string()))
282 }
283
284 fn extract_wind_direction(document: &Html) -> Option<WindDirection> {
285 let p_selector = Selector::parse("p").unwrap();
286
287 for element in document.select(&p_selector) {
288 let text = element.text().collect::<String>();
289
290 if text.to_lowercase().contains("dirección") || text.to_lowercase().contains("direccion") {
291 if let Some(dir) = WindDirection::from_spanish(&text) {
293 return Some(dir);
294 }
295 }
296 }
297
298 let div_selector = Selector::parse("div").unwrap();
300 for element in document.select(&div_selector) {
301 let text = element.text().collect::<String>();
302
303 if (text.contains("viento") || text.contains("Viento")) && text.len() < 100 {
305 if let Some(dir) = WindDirection::from_spanish(&text) {
306 return Some(dir);
307 }
308 }
309 }
310
311 None
312 }
313
314 fn extract_so2_data(document: &Html) -> (Option<f64>, Option<NaiveDate>) {
315 let selector = Selector::parse("div, p").unwrap();
316
317 for element in document.select(&selector) {
318 let text = element.text().collect::<String>();
319
320 if text.contains("toneladas por día") && text.contains("bióxido de azufre") {
321 if let Some(emissions) = Self::extract_so2_value(&text) {
323 let date = Self::extract_so2_date(&text);
324 return (Some(emissions), date);
325 }
326 }
327 }
328
329 (None, None)
330 }
331
332 fn extract_so2_value(text: &str) -> Option<f64> {
333 let words: Vec<&str> = text.split_whitespace().collect();
334 for (i, word) in words.iter().enumerate() {
335 if *word == "toneladas" && i > 0 {
336 if let Ok(val) = words[i - 1].parse::<f64>() {
337 return Some(val);
338 }
339 }
340 }
341 None
342 }
343
344 fn extract_so2_date(text: &str) -> Option<NaiveDate> {
345 let words: Vec<&str> = text.split_whitespace().collect();
347
348 for i in 0..words.len().saturating_sub(2) {
349 if let Ok(day) = words[i].parse::<u32>() {
350 if let Some(month) = Self::parse_spanish_month(words[i + 1]) {
351 if let Ok(year) = words[i + 2].parse::<i32>() {
352 if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
353 return Some(date);
354 }
355 }
356 }
357 }
358 }
359
360 None
361 }
362
363 pub fn parse_spanish_month(text: &str) -> Option<u32> {
364 match text.to_lowercase().as_str() {
365 "enero" => Some(1),
366 "febrero" => Some(2),
367 "marzo" => Some(3),
368 "abril" => Some(4),
369 "mayo" => Some(5),
370 "junio" => Some(6),
371 "julio" => Some(7),
372 "agosto" => Some(8),
373 "septiembre" => Some(9),
374 "octubre" => Some(10),
375 "noviembre" => Some(11),
376 "diciembre" => Some(12),
377 _ => None,
378 }
379 }
380
381 fn extract_summary(document: &Html) -> Result<String> {
382 let selector = Selector::parse("p").unwrap();
383 let mut summary_parts = Vec::new();
384
385 for element in document.select(&selector) {
386 let text = element.text().collect::<String>().trim().to_string();
387
388 if text.contains("exhalaciones") || text.contains("tremor") ||
389 text.contains("actividad") {
390 if !text.is_empty() && text.len() > 20 {
391 summary_parts.push(text);
392 if summary_parts.len() >= 3 {
393 break;
394 }
395 }
396 }
397 }
398
399 if summary_parts.is_empty() {
400 for element in document.select(&selector) {
402 let text = element.text().collect::<String>().trim().to_string();
403 if !text.is_empty() && text.len() > 50 {
404 summary_parts.push(text);
405 if summary_parts.len() >= 3 {
406 break;
407 }
408 }
409 }
410 }
411
412 if summary_parts.is_empty() {
413 Ok("No summary available".to_string())
414 } else {
415 Ok(summary_parts.join("\n\n"))
416 }
417 }
418
419 fn extract_image_urls(document: &Html) -> Vec<String> {
420 let selector = Selector::parse("a[href*='/media/']").unwrap();
421 let mut urls = Vec::new();
422
423 for element in document.select(&selector) {
424 if let Some(href) = element.value().attr("href") {
425 if href.ends_with(".jpg") || href.ends_with(".png") {
426 let full_url = if href.starts_with("http") {
427 href.to_string()
428 } else {
429 format!("{}{}", BASE_URL, href)
430 };
431 urls.push(full_url);
432 }
433 }
434 }
435
436 urls
437 }
438
439 fn extract_video_urls(document: &Html) -> Vec<String> {
440 let selector = Selector::parse("a[href*='/media/']").unwrap();
441 let mut urls = Vec::new();
442
443 for element in document.select(&selector) {
444 if let Some(href) = element.value().attr("href") {
445 if href.ends_with(".mp4") || href.ends_with(".webm") {
446 let full_url = if href.starts_with("http") {
447 href.to_string()
448 } else {
449 format!("{}{}", BASE_URL, href)
450 };
451 urls.push(full_url);
452 }
453 }
454 }
455
456 urls
457 }
458}
459
460impl Default for Scraper {
461 fn default() -> Self {
462 Self::new()
463 }
464}
465
466#[cfg(test)]
467mod tests {
468 use super::*;
469
470 #[test]
471 fn test_parse_spanish_date() {
472 let result = Scraper::parse_spanish_date("06 de Octubre de 2025").unwrap();
473 assert_eq!(result, NaiveDate::from_ymd_opt(2025, 10, 6).unwrap());
474
475 let result = Scraper::parse_spanish_date("15 de Enero de 2024").unwrap();
476 assert_eq!(result, NaiveDate::from_ymd_opt(2024, 1, 15).unwrap());
477 }
478
479 #[test]
480 fn test_parse_report_with_fixture() {
481 let html = include_str!("../tests/fixtures/minimal_report.html");
482 let scraper = Scraper::new();
483
484 let report = scraper.parse_report(html).unwrap();
485
486 assert_eq!(report.date, NaiveDate::from_ymd_opt(2025, 10, 6).unwrap());
487 assert_eq!(report.exhalations, 15);
488 assert_eq!(report.volcanotectonic_events, 0);
489 assert_eq!(report.tremor_minutes_total, 53);
490 assert_eq!(report.explosions, 0);
491 assert_eq!(report.alert_level, AlertLevel::Yellow);
492 assert!(report.alert_phase.contains("AMARILLO"));
493
494 assert_eq!(report.tremor_high_frequency_minutes, Some(39));
496 assert_eq!(report.tremor_harmonic_minutes, Some(14));
497
498 assert_eq!(report.so2_emissions_tons_per_day, Some(2603.0));
500
501 assert!(!report.summary_spanish.is_empty());
503 assert!(report.summary_spanish.contains("exhalaciones"));
504 }
505
506 #[test]
507 fn test_extract_number_before() {
508 let text = "se registraron 39 minutos fueron de alta frecuencia";
509 let result = Scraper::extract_number_before(text, "minutos fueron de alta frecuencia");
510 assert_eq!(result, Some(39));
511 }
512
513 #[test]
514 fn test_parse_spanish_month() {
515 assert_eq!(Scraper::parse_spanish_month("enero"), Some(1));
516 assert_eq!(Scraper::parse_spanish_month("octubre"), Some(10));
517 assert_eq!(Scraper::parse_spanish_month("diciembre"), Some(12));
518 assert_eq!(Scraper::parse_spanish_month("invalid"), None);
519 }
520}