bms_table/
fetch.rs

1//! Data fetching and HTML parsing helpers
2//!
3//! Provides HTML parsing when the `scraper` feature is enabled, used to extract the header JSON URL from
4//! `<meta name="bmstable" content="...">` in a page.
5//! Also provides a unified entry to parse a response string into the header JSON or its URL.
6//!
7//! # Examples
8//!
9//! ```rust
10//! # use bms_table::fetch::{get_web_header_json_value, HeaderQueryContent};
11//! let html = r#"
12//! <!DOCTYPE html>
13//! <html>
14//!   <head>
15//!     <meta name="bmstable" content="header.json">
16//!   </head>
17//!   <body></body>
18//! </html>
19//! "#;
20//! match get_web_header_json_value(html).unwrap() {
21//!     HeaderQueryContent::Url(u) => assert_eq!(u, "header.json"),
22//!     _ => unreachable!(),
23//! }
24//! ```
25#![cfg(feature = "scraper")]
26
27pub mod reqwest;
28
29use anyhow::{Result, anyhow};
30use scraper::{Html, Selector};
31use serde_json::Value;
32
33/// Return type of [`get_web_header_json_value`].
34///
35/// - If the input is HTML, returns the URL extracted from `<meta name="bmstable">`;
36/// - If the input is JSON, returns the parsed JSON value.
37pub enum HeaderQueryContent {
38    /// Extracted header JSON URL.
39    ///
40    /// May be relative or absolute; prefer using `url::Url::join` to resolve.
41    Url(String),
42    /// Raw header JSON content.
43    Json(Value),
44}
45
46/// Remove non-printable control characters from JSON text (preserves `\n`, `\r`, `\t`).
47///
48/// Rationale: some sites return JSON with illegal control characters surrounding it.
49/// Cleaning prior to parsing improves compatibility while not affecting preservation of raw text.
50pub(crate) fn replace_control_chars(s: &str) -> String {
51    s.chars().filter(|ch: &char| !ch.is_control()).collect()
52}
53
54/// Parse a response string into the header JSON or its URL.
55///
56/// Strategy: first attempt to parse as JSON; if it fails, parse as HTML and extract the bmstable URL.
57///
58/// # Returns
59///
60/// - `HeaderQueryContent::Json`: input is JSON;
61/// - `HeaderQueryContent::Url`: input is HTML.
62///
63/// # Errors
64///
65/// Returns an error when the input is HTML but the bmstable field cannot be found.
66pub fn get_web_header_json_value(response_str: &str) -> anyhow::Result<HeaderQueryContent> {
67    // First try parsing as JSON (remove illegal control characters before parsing); if it fails, treat as HTML and extract the bmstable URL
68    let cleaned = replace_control_chars(response_str);
69    match serde_json::from_str::<Value>(&cleaned) {
70        Ok(header_json) => Ok(HeaderQueryContent::Json(header_json)),
71        Err(_) => {
72            let bmstable_url = extract_bmstable_url(response_str)?;
73            Ok(HeaderQueryContent::Url(bmstable_url))
74        }
75    }
76}
77
78/// Extract the JSON file URL pointed to by the bmstable field from HTML page content.
79///
80/// Scans `<meta>` tags looking for elements with `name="bmstable"` and reads their `content` attribute.
81///
82/// # Errors
83///
84/// Returns an error when the target tag is not found or `content` is empty.
85pub fn extract_bmstable_url(html_content: &str) -> Result<String> {
86    let document = Html::parse_document(html_content);
87
88    // Find all meta tags
89    let Ok(meta_selector) = Selector::parse("meta") else {
90        return Err(anyhow!("meta tag not found"));
91    };
92
93    // 1) Prefer extracting from <meta name="bmstable" content="..."> or <meta property="bmstable">
94    for element in document.select(&meta_selector) {
95        // Tags whose name or property is bmstable
96        let is_bmstable = element
97            .value()
98            .attr("name")
99            .is_some_and(|v| v.eq_ignore_ascii_case("bmstable"))
100            || element
101                .value()
102                .attr("property")
103                .is_some_and(|v| v.eq_ignore_ascii_case("bmstable"));
104        if is_bmstable
105            && let Some(content_attr) = element.value().attr("content")
106            && !content_attr.is_empty()
107        {
108            return Ok(content_attr.to_string());
109        }
110    }
111
112    // 2) Next, try <link rel="bmstable" href="...json">
113    if let Ok(link_selector) = Selector::parse("link") {
114        for element in document.select(&link_selector) {
115            let rel = element.value().attr("rel");
116            let href = element.value().attr("href");
117            if rel.is_some_and(|v| v.eq_ignore_ascii_case("bmstable"))
118                && let Some(href) = href
119                && !href.is_empty()
120            {
121                return Ok(href.to_string());
122            }
123        }
124    }
125
126    // 3) Then try to find clues for *header*.json in common tag attributes
127    //    - a[href], link[href], script[src], meta[content]
128    let lower_contains_header_json = |s: &str| {
129        let ls = s.to_ascii_lowercase();
130        ls.contains("header") && ls.ends_with(".json")
131    };
132
133    // a[href]
134    if let Ok(a_selector) = Selector::parse("a") {
135        for element in document.select(&a_selector) {
136            if let Some(href) = element.value().attr("href")
137                && lower_contains_header_json(href)
138            {
139                return Ok(href.to_string());
140            }
141        }
142    }
143
144    // link[href]
145    if let Ok(link_selector) = Selector::parse("link") {
146        for element in document.select(&link_selector) {
147            if let Some(href) = element.value().attr("href")
148                && lower_contains_header_json(href)
149            {
150                return Ok(href.to_string());
151            }
152        }
153    }
154
155    // script[src]
156    if let Ok(script_selector) = Selector::parse("script") {
157        for element in document.select(&script_selector) {
158            if let Some(src) = element.value().attr("src")
159                && lower_contains_header_json(src)
160            {
161                return Ok(src.to_string());
162            }
163        }
164    }
165
166    // meta[content]
167    for element in document.select(&meta_selector) {
168        if let Some(content_attr) = element.value().attr("content")
169            && lower_contains_header_json(content_attr)
170        {
171            return Ok(content_attr.to_string());
172        }
173    }
174
175    // 4) Finally, a minimal heuristic search on raw text: match substrings containing "header" and ending with .json
176    if let Some((start, end)) = find_header_json_in_text(html_content) {
177        let candidate = &html_content[start..end];
178        return Ok(candidate.to_string());
179    }
180
181    Err(anyhow!("bmstable field or header JSON hint not found"))
182}
183
184/// Find a substring like "*header*.json" in raw text, returning start/end indices if found.
185fn find_header_json_in_text(s: &str) -> Option<(usize, usize)> {
186    let lower = s.to_ascii_lowercase();
187    let mut pos = 0;
188    while let Some(idx) = lower[pos..].find("header") {
189        let global_idx = pos + idx;
190        // Look for .json after header
191        if let Some(json_rel) = lower[global_idx..].find(".json") {
192            let end = global_idx + json_rel + ".json".len();
193            // Try to find the nearest quote or whitespace before as the start
194            let start = lower[..global_idx]
195                .rfind(|c: char| c == '"' || c == '\'' || c.is_whitespace())
196                .map(|i| i + 1)
197                .unwrap_or(global_idx);
198            if end > start {
199                return Some((start, end));
200            }
201        }
202        pos = global_idx + 6; // skip "header"
203    }
204    None
205}
bms_table/fetch.rs

bms_table/
fetch.rs
