Skip to main content

popo_cli/
scraper.rs

1use crate::error::{PopoError, Result};
2use crate::models::{AlertLevel, VolcanoReport, WindDirection};
3use chrono::{Datelike, NaiveDate, Utc};
4use scraper::{Html, Selector};
5
6const BASE_URL: &str = "https://www.cenapred.unam.mx";
7const REPORT_URL: &str = "https://www.cenapred.unam.mx/reportesVolcanesMX/Procesos?tipoProceso=detallesUltimoReporteVolcan";
8
9pub struct Scraper {
10    client: reqwest::blocking::Client,
11}
12
13impl Scraper {
14    pub fn new() -> Self {
15        let client = reqwest::blocking::Client::builder()
16            .user_agent("popo-cli/0.1.0 (+https://github.com/KyleEdwardDonaldson/popo-cli)")
17            .timeout(std::time::Duration::from_secs(30))
18            .build()
19            .expect("Failed to create HTTP client");
20
21        Self { client }
22    }
23
24    pub fn fetch_latest(&self) -> Result<VolcanoReport> {
25        let html = self.client.get(REPORT_URL).send()?.text()?;
26        self.parse_report(&html)
27    }
28
29    pub fn fetch_date(&self, date: NaiveDate) -> Result<VolcanoReport> {
30        // URL pattern: https://www.cenapred.unam.mx/reportesVolcanesMX/Procesos?fecha=YYYY-MM-DD
31        let url = format!(
32            "https://www.cenapred.unam.mx/reportesVolcanesMX/Procesos?fecha={}",
33            date.format("%Y-%m-%d")
34        );
35
36        let response = self.client.get(&url).send()?;
37        let html = response.text()?;
38        let report = self.parse_report(&html)?;
39
40        // Verify the date matches what we requested
41        if report.date == date {
42            Ok(report)
43        } else {
44            Err(PopoError::Parse(format!(
45                "Report date mismatch: requested {}, got {}",
46                date, report.date
47            )))
48        }
49    }
50
51    pub fn parse_report(&self, html: &str) -> Result<VolcanoReport> {
52        let document = Html::parse_document(html);
53
54        // Extract date from heading
55        let date = Self::extract_date(&document)?;
56
57        // Extract data from tables
58        let exhalations = Self::extract_from_table(&document, "Exhalaciones", &date)?;
59        let volcanotectonic_events = Self::extract_from_table(&document, "Volcanotectónicos", &date)?;
60        let tremor_minutes_total = Self::extract_from_table(&document, "Minutos de tremor", &date)?;
61        let explosions = Self::extract_from_table(&document, "Explosiones", &date)?;
62
63        // Extract tremor breakdown from summary text
64        let (tremor_high_freq, tremor_harmonic) = Self::extract_tremor_breakdown(&document);
65
66        // Extract alert level and phase
67        let (alert_level, alert_phase) = Self::extract_alert_info(&document)?;
68
69        // Extract wind direction
70        let wind_direction = Self::extract_wind_direction(&document);
71
72        // Extract SO2 data
73        let (so2_emissions, so2_date) = Self::extract_so2_data(&document);
74
75        // Extract summary text
76        let summary_spanish = Self::extract_summary(&document)?;
77
78        // Extract media URLs
79        let image_urls = Self::extract_image_urls(&document);
80        let video_urls = Self::extract_video_urls(&document);
81
82        Ok(VolcanoReport {
83            date,
84            report_time: Utc::now(),
85            exhalations,
86            volcanotectonic_events,
87            tremor_minutes_total,
88            tremor_high_frequency_minutes: tremor_high_freq,
89            tremor_harmonic_minutes: tremor_harmonic,
90            explosions,
91            so2_emissions_tons_per_day: so2_emissions,
92            so2_measurement_date: so2_date,
93            alert_level,
94            alert_phase,
95            wind_direction,
96            summary_spanish,
97            image_urls,
98            video_urls,
99            source_url: REPORT_URL.to_string(),
100            scraped_at: Utc::now(),
101        })
102    }
103
104    fn extract_date(document: &Html) -> Result<NaiveDate> {
105        let selector = Selector::parse("h4").unwrap();
106
107        for element in document.select(&selector) {
108            let text = element.text().collect::<String>();
109            let trimmed = text.trim();
110
111            // Try parsing "06 de Octubre de 2025" format
112            if trimmed.contains("de") && trimmed.contains("de 20") {
113                if let Ok(date) = Self::parse_spanish_date(trimmed) {
114                    return Ok(date);
115                }
116            }
117
118            // Try parsing ISO date format "2025-10-06"
119            // Look for YYYY-MM-DD pattern in the text
120            for part in trimmed.split(|c: char| c.is_whitespace() || c == '\u{00A0}') {
121                if part.len() >= 10 {
122                    let potential_date = &part[..part.len().min(10)];
123                    if potential_date.chars().filter(|&c| c == '-').count() == 2 {
124                        if let Ok(date) = NaiveDate::parse_from_str(potential_date, "%Y-%m-%d") {
125                            return Ok(date);
126                        }
127                    }
128                }
129            }
130        }
131
132        Err(PopoError::Parse("Could not find date in report".to_string()))
133    }
134
135    pub fn parse_spanish_date(text: &str) -> Result<NaiveDate> {
136        let parts: Vec<&str> = text.split_whitespace().collect();
137        if parts.len() < 5 {
138            return Err(PopoError::Parse(format!("Invalid date format: {}", text)));
139        }
140
141        let day: u32 = parts[0].parse()
142            .map_err(|_| PopoError::Parse(format!("Invalid day: {}", parts[0])))?;
143
144        let month = match parts[2].to_lowercase().as_str() {
145            "enero" => 1,
146            "febrero" => 2,
147            "marzo" => 3,
148            "abril" => 4,
149            "mayo" => 5,
150            "junio" => 6,
151            "julio" => 7,
152            "agosto" => 8,
153            "septiembre" => 9,
154            "octubre" => 10,
155            "noviembre" => 11,
156            "diciembre" => 12,
157            _ => return Err(PopoError::Parse(format!("Invalid month: {}", parts[2]))),
158        };
159
160        let year: i32 = parts[4].parse()
161            .map_err(|_| PopoError::Parse(format!("Invalid year: {}", parts[4])))?;
162
163        NaiveDate::from_ymd_opt(year, month, day)
164            .ok_or_else(|| PopoError::Parse(format!("Invalid date: {}-{}-{}", year, month, day)))
165    }
166
167    fn extract_from_table(document: &Html, table_name: &str, date: &NaiveDate) -> Result<u32> {
168        let date_str = format!("{:02}-{:02}-{}", date.day(), date.month(), date.year());
169
170        // First try HTML tables (for test fixtures)
171        let table_selector = Selector::parse("table").unwrap();
172        let row_selector = Selector::parse("tr").unwrap();
173        let cell_selector = Selector::parse("td").unwrap();
174
175        for table in document.select(&table_selector) {
176            let header_text = table.text().collect::<String>();
177
178            if header_text.contains(table_name) {
179                // Find the row with our date
180                for row in table.select(&row_selector) {
181                    let cells: Vec<_> = row.select(&cell_selector).collect();
182
183                    if cells.len() >= 2 {
184                        let date_cell = cells[0].text().collect::<String>();
185
186                        if date_cell.trim() == date_str {
187                            let value_cell = cells[1].text().collect::<String>();
188                            let value: u32 = value_cell.trim().parse()
189                                .map_err(|_| PopoError::Parse(format!("Invalid value for {}: {}", table_name, value_cell)))?;
190                            return Ok(value);
191                        }
192                    }
193                }
194            }
195        }
196
197        // Try parsing JavaScript data (for live site with Google Charts)
198        let script_selector = Selector::parse("script").unwrap();
199        for script in document.select(&script_selector) {
200            let script_text = script.text().collect::<String>();
201
202            let column_pattern = format!("addColumn('number', '{}')", table_name);
203            if let Some(column_pos) = script_text.find(&column_pattern) {
204                // Look for data.addRows([ ... ]) pattern AFTER the column definition
205                let after_column = &script_text[column_pos..];
206                if let Some(rows_start) = after_column.find("data.addRows([") {
207                    let after_rows_start = &after_column[rows_start + 14..];
208                    if let Some(rows_end) = after_rows_start.find("]);") {
209                        let data_section = &after_rows_start[..rows_end];
210
211                        // Parse each row: ['YYYY-MM-DD...', value, color]
212                        // Look for a line that starts with the target date
213                        let target_date_prefix = format!("'{}-{:02}-{:02}", date.year(), date.month(), date.day());
214                        for line in data_section.lines() {
215                            let trimmed = line.trim();
216                            if trimmed.starts_with('[') && trimmed.contains(&target_date_prefix) {
217                                // Extract the value (second element in array)
218                                // Format: ['2025-10-06...', 15, '#color']
219                                let parts: Vec<&str> = trimmed.split(',').collect();
220                                if parts.len() >= 2 {
221                                    let value_str = parts[1].trim();
222                                    if let Ok(value) = value_str.parse::<u32>() {
223                                        return Ok(value);
224                                    }
225                                }
226                            }
227                        }
228                    }
229                }
230            }
231        }
232
233        Err(PopoError::Parse(format!("Could not find {} data for date {}", table_name, date_str)))
234    }
235
236    fn extract_tremor_breakdown(document: &Html) -> (Option<u32>, Option<u32>) {
237        let p_selector = Selector::parse("p").unwrap();
238
239        for element in document.select(&p_selector) {
240            let text = element.text().collect::<String>();
241
242            // Look for pattern like "39 minutos fueron de alta frecuencia y 14 minutos de armónico"
243            if text.contains("alta frecuencia") && text.contains("armónico") {
244                let high_freq = Self::extract_number_before(&text, "minutos fueron de alta frecuencia");
245                let harmonic = Self::extract_number_before(&text, "minutos de armónico");
246                return (high_freq, harmonic);
247            }
248        }
249
250        (None, None)
251    }
252
253    pub fn extract_number_before(text: &str, marker: &str) -> Option<u32> {
254        if let Some(pos) = text.find(marker) {
255            let before = &text[..pos];
256            let words: Vec<&str> = before.split_whitespace().collect();
257            if let Some(last_word) = words.last() {
258                if let Ok(num) = last_word.parse::<u32>() {
259                    return Some(num);
260                }
261            }
262        }
263        None
264    }
265
266    fn extract_alert_info(document: &Html) -> Result<(AlertLevel, String)> {
267        let selector = Selector::parse("p, h4").unwrap();
268
269        for element in document.select(&selector) {
270            let text = element.text().collect::<String>();
271
272            if text.contains("AMARILLO") || text.contains("VERDE") ||
273               text.contains("ROJO") || text.contains("NARANJA") {
274                if let Some(level) = AlertLevel::from_spanish(&text) {
275                    let phase = text.trim().to_string();
276                    return Ok((level, phase));
277                }
278            }
279        }
280
281        Err(PopoError::Parse("Could not find alert level".to_string()))
282    }
283
284    fn extract_wind_direction(document: &Html) -> Option<WindDirection> {
285        let p_selector = Selector::parse("p").unwrap();
286
287        for element in document.select(&p_selector) {
288            let text = element.text().collect::<String>();
289
290            if text.to_lowercase().contains("dirección") || text.to_lowercase().contains("direccion") {
291                // Look for direction keywords
292                if let Some(dir) = WindDirection::from_spanish(&text) {
293                    return Some(dir);
294                }
295            }
296        }
297
298        // Also check div elements which might contain wind direction
299        let div_selector = Selector::parse("div").unwrap();
300        for element in document.select(&div_selector) {
301            let text = element.text().collect::<String>();
302
303            // Look in elements that might contain direction info
304            if (text.contains("viento") || text.contains("Viento")) && text.len() < 100 {
305                if let Some(dir) = WindDirection::from_spanish(&text) {
306                    return Some(dir);
307                }
308            }
309        }
310
311        None
312    }
313
314    fn extract_so2_data(document: &Html) -> (Option<f64>, Option<NaiveDate>) {
315        let selector = Selector::parse("div, p").unwrap();
316
317        for element in document.select(&selector) {
318            let text = element.text().collect::<String>();
319
320            if text.contains("toneladas por día") && text.contains("bióxido de azufre") {
321                // Extract number before "toneladas por día"
322                if let Some(emissions) = Self::extract_so2_value(&text) {
323                    let date = Self::extract_so2_date(&text);
324                    return (Some(emissions), date);
325                }
326            }
327        }
328
329        (None, None)
330    }
331
332    fn extract_so2_value(text: &str) -> Option<f64> {
333        let words: Vec<&str> = text.split_whitespace().collect();
334        for (i, word) in words.iter().enumerate() {
335            if *word == "toneladas" && i > 0 {
336                if let Ok(val) = words[i - 1].parse::<f64>() {
337                    return Some(val);
338                }
339            }
340        }
341        None
342    }
343
344    fn extract_so2_date(text: &str) -> Option<NaiveDate> {
345        // Look for date pattern like "10 octubre 2024"
346        let words: Vec<&str> = text.split_whitespace().collect();
347
348        for i in 0..words.len().saturating_sub(2) {
349            if let Ok(day) = words[i].parse::<u32>() {
350                if let Some(month) = Self::parse_spanish_month(words[i + 1]) {
351                    if let Ok(year) = words[i + 2].parse::<i32>() {
352                        if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
353                            return Some(date);
354                        }
355                    }
356                }
357            }
358        }
359
360        None
361    }
362
363    pub fn parse_spanish_month(text: &str) -> Option<u32> {
364        match text.to_lowercase().as_str() {
365            "enero" => Some(1),
366            "febrero" => Some(2),
367            "marzo" => Some(3),
368            "abril" => Some(4),
369            "mayo" => Some(5),
370            "junio" => Some(6),
371            "julio" => Some(7),
372            "agosto" => Some(8),
373            "septiembre" => Some(9),
374            "octubre" => Some(10),
375            "noviembre" => Some(11),
376            "diciembre" => Some(12),
377            _ => None,
378        }
379    }
380
381    fn extract_summary(document: &Html) -> Result<String> {
382        let selector = Selector::parse("p").unwrap();
383        let mut summary_parts = Vec::new();
384
385        for element in document.select(&selector) {
386            let text = element.text().collect::<String>().trim().to_string();
387
388            if text.contains("exhalaciones") || text.contains("tremor") ||
389               text.contains("actividad") {
390                if !text.is_empty() && text.len() > 20 {
391                    summary_parts.push(text);
392                    if summary_parts.len() >= 3 {
393                        break;
394                    }
395                }
396            }
397        }
398
399        if summary_parts.is_empty() {
400            // If no summary found with strict criteria, try to find any paragraph with substantial text
401            for element in document.select(&selector) {
402                let text = element.text().collect::<String>().trim().to_string();
403                if !text.is_empty() && text.len() > 50 {
404                    summary_parts.push(text);
405                    if summary_parts.len() >= 3 {
406                        break;
407                    }
408                }
409            }
410        }
411
412        if summary_parts.is_empty() {
413            Ok("No summary available".to_string())
414        } else {
415            Ok(summary_parts.join("\n\n"))
416        }
417    }
418
419    fn extract_image_urls(document: &Html) -> Vec<String> {
420        let selector = Selector::parse("a[href*='/media/']").unwrap();
421        let mut urls = Vec::new();
422
423        for element in document.select(&selector) {
424            if let Some(href) = element.value().attr("href") {
425                if href.ends_with(".jpg") || href.ends_with(".png") {
426                    let full_url = if href.starts_with("http") {
427                        href.to_string()
428                    } else {
429                        format!("{}{}", BASE_URL, href)
430                    };
431                    urls.push(full_url);
432                }
433            }
434        }
435
436        urls
437    }
438
439    fn extract_video_urls(document: &Html) -> Vec<String> {
440        let selector = Selector::parse("a[href*='/media/']").unwrap();
441        let mut urls = Vec::new();
442
443        for element in document.select(&selector) {
444            if let Some(href) = element.value().attr("href") {
445                if href.ends_with(".mp4") || href.ends_with(".webm") {
446                    let full_url = if href.starts_with("http") {
447                        href.to_string()
448                    } else {
449                        format!("{}{}", BASE_URL, href)
450                    };
451                    urls.push(full_url);
452                }
453            }
454        }
455
456        urls
457    }
458}
459
460impl Default for Scraper {
461    fn default() -> Self {
462        Self::new()
463    }
464}
465
466#[cfg(test)]
467mod tests {
468    use super::*;
469
470    #[test]
471    fn test_parse_spanish_date() {
472        let result = Scraper::parse_spanish_date("06 de Octubre de 2025").unwrap();
473        assert_eq!(result, NaiveDate::from_ymd_opt(2025, 10, 6).unwrap());
474
475        let result = Scraper::parse_spanish_date("15 de Enero de 2024").unwrap();
476        assert_eq!(result, NaiveDate::from_ymd_opt(2024, 1, 15).unwrap());
477    }
478
479    #[test]
480    fn test_parse_report_with_fixture() {
481        let html = include_str!("../tests/fixtures/minimal_report.html");
482        let scraper = Scraper::new();
483
484        let report = scraper.parse_report(html).unwrap();
485
486        assert_eq!(report.date, NaiveDate::from_ymd_opt(2025, 10, 6).unwrap());
487        assert_eq!(report.exhalations, 15);
488        assert_eq!(report.volcanotectonic_events, 0);
489        assert_eq!(report.tremor_minutes_total, 53);
490        assert_eq!(report.explosions, 0);
491        assert_eq!(report.alert_level, AlertLevel::Yellow);
492        assert!(report.alert_phase.contains("AMARILLO"));
493
494        // Check tremor breakdown
495        assert_eq!(report.tremor_high_frequency_minutes, Some(39));
496        assert_eq!(report.tremor_harmonic_minutes, Some(14));
497
498        // Check SO2
499        assert_eq!(report.so2_emissions_tons_per_day, Some(2603.0));
500
501        // Check that we got some summary text
502        assert!(!report.summary_spanish.is_empty());
503        assert!(report.summary_spanish.contains("exhalaciones"));
504    }
505
506    #[test]
507    fn test_extract_number_before() {
508        let text = "se registraron 39 minutos fueron de alta frecuencia";
509        let result = Scraper::extract_number_before(text, "minutos fueron de alta frecuencia");
510        assert_eq!(result, Some(39));
511    }
512
513    #[test]
514    fn test_parse_spanish_month() {
515        assert_eq!(Scraper::parse_spanish_month("enero"), Some(1));
516        assert_eq!(Scraper::parse_spanish_month("octubre"), Some(10));
517        assert_eq!(Scraper::parse_spanish_month("diciembre"), Some(12));
518        assert_eq!(Scraper::parse_spanish_month("invalid"), None);
519    }
520}