hpx_browser/
resource_loader.rs

1use std::collections::HashSet;
2
3use ahash::AHashSet;
4
5use crate::dom::{Dom, NodeId};
6
7#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
8pub enum ResourceType {
9    Stylesheet,
10    Script,
11    Image,
12    Font,
13    Media,
14}
15
16#[derive(Debug, Clone, PartialEq, Eq)]
17pub struct ResourceUrl {
18    pub resource_type: ResourceType,
19    pub url: String,
20}
21
22fn get_attr<'a>(attrs: &'a [crate::dom::Attribute], name: &str) -> Option<&'a str> {
23    attrs
24        .iter()
25        .find(|a| a.name.local.eq_ignore_ascii_case(name))
26        .map(|a| a.value.as_str())
27}
28
29fn extract_from_element(dom: &Dom, id: NodeId) -> Option<ResourceUrl> {
30    let node = dom.get(id)?;
31    let elem = node.as_element()?;
32    let tag = elem.name.local.as_str();
33    match tag {
34        "link" => {
35            let rel = get_attr(&elem.attrs, "rel").unwrap_or("");
36            let href = get_attr(&elem.attrs, "href")?;
37            if rel.eq_ignore_ascii_case("stylesheet") {
38                Some(ResourceUrl {
39                    resource_type: ResourceType::Stylesheet,
40                    url: href.to_owned(),
41                })
42            } else if rel.eq_ignore_ascii_case("preload") {
43                let as_type = get_attr(&elem.attrs, "as").unwrap_or("");
44                let rt = if as_type.eq_ignore_ascii_case("font") {
45                    ResourceType::Font
46                } else if as_type.eq_ignore_ascii_case("style") {
47                    ResourceType::Stylesheet
48                } else if as_type.eq_ignore_ascii_case("script") {
49                    ResourceType::Script
50                } else if as_type.eq_ignore_ascii_case("image") {
51                    ResourceType::Image
52                } else {
53                    return None;
54                };
55                Some(ResourceUrl {
56                    resource_type: rt,
57                    url: href.to_owned(),
58                })
59            } else {
60                None
61            }
62        }
63        "script" => {
64            let src = get_attr(&elem.attrs, "src")?;
65            Some(ResourceUrl {
66                resource_type: ResourceType::Script,
67                url: src.to_owned(),
68            })
69        }
70        "img" => {
71            let src = get_attr(&elem.attrs, "src")?;
72            Some(ResourceUrl {
73                resource_type: ResourceType::Image,
74                url: src.to_owned(),
75            })
76        }
77        "video" | "audio" => {
78            let src = get_attr(&elem.attrs, "src")?;
79            Some(ResourceUrl {
80                resource_type: ResourceType::Media,
81                url: src.to_owned(),
82            })
83        }
84        _ => None,
85    }
86}
87
88/// Extract all resource URLs from the DOM.
89#[cfg_attr(feature = "hotpath", hotpath::measure)]
90pub fn extract_resource_urls(dom: &Dom) -> Vec<ResourceUrl> {
91    let mut results = Vec::new();
92    let mut seen = AHashSet::new();
93    let mut stack = vec![dom.document()];
94    let mut visited = AHashSet::new();
95
96    while let Some(id) = stack.pop() {
97        if !visited.insert(id) {
98            continue;
99        }
100        if let Some(resource) = extract_from_element(dom, id) {
101            if seen.insert(resource.url.clone()) {
102                results.push(resource);
103            }
104        }
105        // push children in reverse so we visit in document order
106        let children = dom.children(id);
107        for child in children.into_iter().rev() {
108            stack.push(child);
109        }
110    }
111
112    results
113}
114
115/// A fetched resource with its content and metadata.
116#[derive(Debug, Clone)]
117pub struct LoadedResource {
118    pub url: String,
119    pub resource_type: ResourceType,
120    pub content: String,
121    pub content_type: Option<String>,
122}
123
124/// Fetch all resources concurrently.
125///
126/// Respects `block_types` — blocked resources are skipped.
127/// Caps concurrent requests at `max_concurrent` (default 6).
128/// Individual resource timeout: 5s. Total timeout: 15s.
129pub async fn fetch_resources(
130    urls: Vec<ResourceUrl>,
131    block_types: &HashSet<ResourceType>,
132    max_concurrent: usize,
133) -> Vec<LoadedResource> {
134    let filtered = filter_by_block_types(urls, block_types);
135    if filtered.is_empty() {
136        return Vec::new();
137    }
138
139    let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent));
140    let client = hpx::Client::new();
141
142    let total = tokio::time::timeout(std::time::Duration::from_secs(15), async {
143        let mut set = tokio::task::JoinSet::new();
144
145        for resource in filtered {
146            let sem = semaphore.clone();
147            let client = client.clone();
148            set.spawn(async move {
149                let _permit = match sem.acquire().await {
150                    Ok(p) => p,
151                    Err(_) => return None,
152                };
153                let result = tokio::time::timeout(
154                    std::time::Duration::from_secs(5),
155                    client.get(&resource.url).send(),
156                )
157                .await;
158
159                match result {
160                    Ok(Ok(resp)) => {
161                        let content_type = resp
162                            .headers()
163                            .get("content-type")
164                            .and_then(|v| v.to_str().ok())
165                            .map(|s| s.to_string());
166                        let url = resp.uri().to_string();
167                        match resp.text().await {
168                            Ok(text) => Some(LoadedResource {
169                                url,
170                                resource_type: resource.resource_type,
171                                content: text,
172                                content_type,
173                            }),
174                            Err(_) => None,
175                        }
176                    }
177                    _ => None,
178                }
179            });
180        }
181
182        let mut results = Vec::new();
183        while let Some(res) = set.join_next().await {
184            if let Ok(Some(loaded)) = res {
185                results.push(loaded);
186            }
187        }
188        results
189    })
190    .await;
191
192    total.unwrap_or_default()
193}
194
195/// Filter resources by blocked types.
196pub fn filter_by_block_types(
197    resources: Vec<ResourceUrl>,
198    block_types: &HashSet<ResourceType>,
199) -> Vec<ResourceUrl> {
200    resources
201        .into_iter()
202        .filter(|r| !block_types.contains(&r.resource_type))
203        .collect()
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209    use crate::html_parser::parse_html;
210
211    #[test]
212    fn extract_stylesheet() {
213        let dom = parse_html(
214            r#"<html><head><link rel="stylesheet" href="/style.css"></head><body></body></html>"#,
215        );
216        let urls = extract_resource_urls(&dom);
217        assert_eq!(urls.len(), 1);
218        assert_eq!(urls[0].resource_type, ResourceType::Stylesheet);
219        assert_eq!(urls[0].url, "/style.css");
220    }
221
222    #[test]
223    fn extract_script() {
224        let dom =
225            parse_html(r#"<html><head><script src="/app.js"></script></head><body></body></html>"#);
226        let urls = extract_resource_urls(&dom);
227        assert_eq!(urls.len(), 1);
228        assert_eq!(urls[0].resource_type, ResourceType::Script);
229        assert_eq!(urls[0].url, "/app.js");
230    }
231
232    #[test]
233    fn extract_img() {
234        let dom = parse_html(r#"<html><body><img src="/logo.png"></body></html>"#);
235        let urls = extract_resource_urls(&dom);
236        assert_eq!(urls.len(), 1);
237        assert_eq!(urls[0].resource_type, ResourceType::Image);
238        assert_eq!(urls[0].url, "/logo.png");
239    }
240
241    #[test]
242    fn extract_video_and_audio() {
243        let dom = parse_html(
244            r#"<html><body><video src="/v.mp4"></video><audio src="/a.mp3"></audio></body></html>"#,
245        );
246        let urls = extract_resource_urls(&dom);
247        assert_eq!(urls.len(), 2);
248        assert!(urls.iter().all(|r| r.resource_type == ResourceType::Media));
249    }
250
251    #[test]
252    fn extract_preload_font() {
253        let dom = parse_html(
254            r#"<html><head><link rel="preload" href="/font.woff2" as="font" crossorigin></head><body></body></html>"#,
255        );
256        let urls = extract_resource_urls(&dom);
257        assert_eq!(urls.len(), 1);
258        assert_eq!(urls[0].resource_type, ResourceType::Font);
259        assert_eq!(urls[0].url, "/font.woff2");
260    }
261
262    #[test]
263    fn inline_script_ignored() {
264        let dom = parse_html(r#"<html><head><script>alert(1)</script></head><body></body></html>"#);
265        let urls = extract_resource_urls(&dom);
266        assert!(urls.is_empty());
267    }
268
269    #[test]
270    fn all_resource_types() {
271        let dom = parse_html(
272            r#"<html>
273            <head>
274                <link rel="stylesheet" href="/style.css">
275                <script src="/app.js"></script>
276                <link rel="preload" href="/font.woff2" as="font">
277            </head>
278            <body>
279                <img src="/photo.jpg">
280                <video src="/clip.mp4"></video>
281            </body>
282            </html>"#,
283        );
284        let urls = extract_resource_urls(&dom);
285        assert_eq!(urls.len(), 5);
286        let types: HashSet<ResourceType> = urls.iter().map(|r| r.resource_type).collect();
287        assert!(types.contains(&ResourceType::Stylesheet));
288        assert!(types.contains(&ResourceType::Script));
289        assert!(types.contains(&ResourceType::Image));
290        assert!(types.contains(&ResourceType::Font));
291        assert!(types.contains(&ResourceType::Media));
292    }
293
294    #[test]
295    fn dedup_same_url() {
296        let dom = parse_html(
297            r#"<html><head>
298                <link rel="stylesheet" href="/style.css">
299                <link rel="stylesheet" href="/style.css">
300            </head><body></body></html>"#,
301        );
302        let urls = extract_resource_urls(&dom);
303        assert_eq!(urls.len(), 1);
304    }
305
306    #[test]
307    fn filter_blocks_stylesheets() {
308        let dom = parse_html(
309            r#"<html><head>
310                <link rel="stylesheet" href="/style.css">
311                <script src="/app.js"></script>
312            </head><body><img src="/logo.png"></body></html>"#,
313        );
314        let urls = extract_resource_urls(&dom);
315        let mut block = HashSet::new();
316        block.insert(ResourceType::Stylesheet);
317        let filtered = filter_by_block_types(urls, &block);
318        assert_eq!(filtered.len(), 2);
319        assert!(
320            filtered
321                .iter()
322                .all(|r| r.resource_type != ResourceType::Stylesheet)
323        );
324    }
325
326    #[test]
327    fn filter_empty_block_returns_all() {
328        let dom = parse_html(
329            r#"<html><head><link rel="stylesheet" href="/s.css"></head><body><img src="/i.png"></body></html>"#,
330        );
331        let urls = extract_resource_urls(&dom);
332        let block = HashSet::new();
333        let filtered = filter_by_block_types(urls.clone(), &block);
334        assert_eq!(filtered.len(), urls.len());
335    }
336
337    #[test]
338    fn filter_all_blocks_returns_empty() {
339        let dom = parse_html(
340            r#"<html><head>
341                <link rel="stylesheet" href="/style.css">
342                <script src="/app.js"></script>
343            </head><body><img src="/logo.png"></body></html>"#,
344        );
345        let urls = extract_resource_urls(&dom);
346        let mut block = HashSet::new();
347        block.insert(ResourceType::Stylesheet);
348        block.insert(ResourceType::Script);
349        block.insert(ResourceType::Image);
350        block.insert(ResourceType::Font);
351        block.insert(ResourceType::Media);
352        let filtered = filter_by_block_types(urls, &block);
353        assert!(filtered.is_empty());
354    }
355
356    #[test]
357    fn filter_idempotent() {
358        let dom = parse_html(
359            r#"<html><head>
360                <link rel="stylesheet" href="/style.css">
361                <script src="/app.js"></script>
362            </head><body><img src="/logo.png"></body></html>"#,
363        );
364        let urls = extract_resource_urls(&dom);
365        let mut block = HashSet::new();
366        block.insert(ResourceType::Stylesheet);
367        let once = filter_by_block_types(urls.clone(), &block);
368        let twice = filter_by_block_types(once.clone(), &block);
369        assert_eq!(once, twice);
370    }
371
372    #[test]
373    fn block_images_from_html() {
374        let dom =
375            parse_html(r#"<html><body><img src="/photo.jpg"><img src="/icon.png"></body></html>"#);
376        let urls = extract_resource_urls(&dom);
377        assert_eq!(urls.len(), 2);
378        assert!(urls.iter().all(|r| r.resource_type == ResourceType::Image));
379        let mut block = HashSet::new();
380        block.insert(ResourceType::Image);
381        let filtered = filter_by_block_types(urls, &block);
382        assert!(filtered.is_empty());
383    }
384
385    #[test]
386    fn no_resources() {
387        let dom = parse_html(r#"<html><body><p>Hello</p></body></html>"#);
388        let urls = extract_resource_urls(&dom);
389        assert!(urls.is_empty());
390    }
391
392    #[tokio::test]
393    async fn fetch_resources_concurrent() {
394        use tokio::io::AsyncWriteExt;
395
396        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
397        let addr = listener.local_addr().unwrap();
398        let base = format!("http://{addr}");
399
400        let resources = vec![
401            ("/style.css", "text/css", "body{}"),
402            ("/app.js", "application/javascript", "alert(1)"),
403            ("/logo.png", "image/png", "PNGDATA"),
404        ];
405
406        let server_resources = resources.clone();
407        let server = tokio::spawn(async move {
408            for (_path, ct, body) in &server_resources {
409                let (mut stream, _) = listener.accept().await.unwrap();
410                let response = format!(
411                    "HTTP/1.1 200 OK\r\nContent-Type: {ct}\r\nContent-Length: {}\r\n\r\n{body}",
412                    body.len()
413                );
414                stream.write_all(response.as_bytes()).await.unwrap();
415            }
416        });
417
418        let urls: Vec<ResourceUrl> = resources
419            .iter()
420            .map(|(path, _, _)| ResourceUrl {
421                resource_type: if path.ends_with(".css") {
422                    ResourceType::Stylesheet
423                } else if path.ends_with(".js") {
424                    ResourceType::Script
425                } else {
426                    ResourceType::Image
427                },
428                url: format!("{base}{path}"),
429            })
430            .collect();
431
432        let loaded = fetch_resources(urls, &HashSet::new(), 6).await;
433        server.await.unwrap();
434
435        assert_eq!(loaded.len(), 3);
436        let mut contents: Vec<&str> = loaded.iter().map(|r| r.content.as_str()).collect();
437        contents.sort();
438        assert_eq!(contents, vec!["PNGDATA", "alert(1)", "body{}"]);
439    }
440
441    #[tokio::test]
442    async fn fetch_resources_respects_block_types() {
443        use tokio::io::AsyncWriteExt;
444
445        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
446        let addr = listener.local_addr().unwrap();
447        let base = format!("http://{addr}");
448
449        let _server = tokio::spawn(async move {
450            let (mut stream, _) = listener.accept().await.unwrap();
451            let body = "body{}";
452            let response = format!(
453                "HTTP/1.1 200 OK\r\nContent-Type: text/css\r\nContent-Length: {}\r\n\r\n{body}",
454                body.len()
455            );
456            stream.write_all(response.as_bytes()).await.unwrap();
457        });
458
459        let urls = vec![ResourceUrl {
460            resource_type: ResourceType::Stylesheet,
461            url: format!("{base}/style.css"),
462        }];
463
464        let mut block = HashSet::new();
465        block.insert(ResourceType::Stylesheet);
466        let loaded = fetch_resources(urls, &block, 6).await;
467
468        assert!(loaded.is_empty());
469        // server never got a connection since it was blocked
470    }
471
472    #[tokio::test]
473    async fn fetch_resources_empty_input() {
474        let loaded = fetch_resources(vec![], &HashSet::new(), 6).await;
475        assert!(loaded.is_empty());
476    }
477}
478
479#[cfg(test)]
480#[cfg(feature = "proptest")]
481mod proptests {
482    use proptest::prelude::*;
483
484    use super::*;
485    use crate::html_parser::parse_html;
486
487    fn url_strategy() -> impl Strategy<Value = String> {
488        // First char is always a letter to avoid protocol-relative URLs ("//")
489        // that cause blitz-dom to panic during URL resolution.
490        ("[a-zA-Z]", prop::collection::vec("[a-zA-Z0-9/._-]", 0..19))
491            .prop_map(|(first, rest)| format!("{}{}", first, rest.join("")))
492    }
493
494    fn resource_tag_strategy() -> impl Strategy<Value = String> {
495        prop_oneof![
496            // link stylesheet
497            url_strategy().prop_map(|href| format!(r#"<link rel="stylesheet" href="{}">"#, href)),
498            // script src
499            url_strategy().prop_map(|src| format!(r#"<script src="{}"></script>"#, src)),
500            // img src
501            url_strategy().prop_map(|src| format!(r#"<img src="{}">"#, src)),
502            // video src
503            url_strategy().prop_map(|src| format!(r#"<video src="{}"></video>"#, src)),
504            // audio src
505            url_strategy().prop_map(|src| format!(r#"<audio src="{}"></audio>"#, src)),
506        ]
507    }
508
509    fn html_with_resources_strategy() -> impl Strategy<Value = String> {
510        prop::collection::vec(resource_tag_strategy(), 0..10).prop_map(|tags| {
511            format!(
512                r#"<html><head>{}</head><body>{}</body></html>"#,
513                tags[..tags.len().min(tags.len() / 2)].join(""),
514                tags[tags.len().min(tags.len() / 2)..].join("")
515            )
516        })
517    }
518
519    proptest! {
520        #[test]
521        fn extracted_urls_are_nonempty_strings(html in html_with_resources_strategy()) {
522            let dom = parse_html(&html);
523            let urls = extract_resource_urls(&dom);
524            for r in &urls {
525                prop_assert!(!r.url.is_empty());
526            }
527        }
528
529        #[test]
530        fn block_filtering_removes_blocked_types(html in html_with_resources_strategy()) {
531            let dom = parse_html(&html);
532            let urls = extract_resource_urls(&dom);
533            let mut block = HashSet::new();
534            block.insert(ResourceType::Stylesheet);
535            let filtered = filter_by_block_types(urls, &block);
536            for r in &filtered {
537                prop_assert_ne!(r.resource_type, ResourceType::Stylesheet);
538            }
539        }
540
541        #[test]
542        fn block_filtering_is_idempotent(html in html_with_resources_strategy()) {
543            let dom = parse_html(&html);
544            let urls = extract_resource_urls(&dom);
545            let mut block = HashSet::new();
546            block.insert(ResourceType::Script);
547            let once = filter_by_block_types(urls.clone(), &block);
548            let twice = filter_by_block_types(once.clone(), &block);
549            prop_assert_eq!(once, twice);
550        }
551    }
552}
hpx_browser/resource_loader.rs

hpx_browser/
resource_loader.rs
