hpx_browser/
resource_loader.rs

1use std::collections::HashSet;
2
3use crate::dom::{Dom, NodeId};
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
6pub enum ResourceType {
7    Stylesheet,
8    Script,
9    Image,
10    Font,
11    Media,
12}
13
14#[derive(Debug, Clone, PartialEq, Eq)]
15pub struct ResourceUrl {
16    pub resource_type: ResourceType,
17    pub url: String,
18}
19
20fn get_attr<'a>(attrs: &'a [crate::dom::Attribute], name: &str) -> Option<&'a str> {
21    attrs
22        .iter()
23        .find(|a| a.name.local.eq_ignore_ascii_case(name))
24        .map(|a| a.value.as_str())
25}
26
27fn extract_from_element(dom: &Dom, id: NodeId) -> Option<ResourceUrl> {
28    let node = dom.get(id)?;
29    let elem = node.as_element()?;
30    let tag = elem.name.local.as_str();
31    match tag {
32        "link" => {
33            let rel = get_attr(&elem.attrs, "rel").unwrap_or("");
34            let href = get_attr(&elem.attrs, "href")?;
35            if rel.eq_ignore_ascii_case("stylesheet") {
36                Some(ResourceUrl {
37                    resource_type: ResourceType::Stylesheet,
38                    url: href.to_owned(),
39                })
40            } else if rel.eq_ignore_ascii_case("preload") {
41                let as_type = get_attr(&elem.attrs, "as").unwrap_or("");
42                let rt = if as_type.eq_ignore_ascii_case("font") {
43                    ResourceType::Font
44                } else if as_type.eq_ignore_ascii_case("style") {
45                    ResourceType::Stylesheet
46                } else if as_type.eq_ignore_ascii_case("script") {
47                    ResourceType::Script
48                } else if as_type.eq_ignore_ascii_case("image") {
49                    ResourceType::Image
50                } else {
51                    return None;
52                };
53                Some(ResourceUrl {
54                    resource_type: rt,
55                    url: href.to_owned(),
56                })
57            } else {
58                None
59            }
60        }
61        "script" => {
62            let src = get_attr(&elem.attrs, "src")?;
63            Some(ResourceUrl {
64                resource_type: ResourceType::Script,
65                url: src.to_owned(),
66            })
67        }
68        "img" => {
69            let src = get_attr(&elem.attrs, "src")?;
70            Some(ResourceUrl {
71                resource_type: ResourceType::Image,
72                url: src.to_owned(),
73            })
74        }
75        "video" | "audio" => {
76            let src = get_attr(&elem.attrs, "src")?;
77            Some(ResourceUrl {
78                resource_type: ResourceType::Media,
79                url: src.to_owned(),
80            })
81        }
82        _ => None,
83    }
84}
85
86/// Extract all resource URLs from the DOM.
87pub fn extract_resource_urls(dom: &Dom) -> Vec<ResourceUrl> {
88    let mut results = Vec::new();
89    let mut seen = HashSet::new();
90    let mut stack = vec![NodeId::DOCUMENT];
91    let mut visited = HashSet::new();
92
93    while let Some(id) = stack.pop() {
94        if !visited.insert(id) {
95            continue;
96        }
97        if let Some(resource) = extract_from_element(dom, id) {
98            if seen.insert(resource.url.clone()) {
99                results.push(resource);
100            }
101        }
102        // push children in reverse so we visit in document order
103        let children = dom.children(id);
104        for child in children.into_iter().rev() {
105            stack.push(child);
106        }
107    }
108
109    results
110}
111
112/// A fetched resource with its content and metadata.
113#[derive(Debug, Clone)]
114pub struct LoadedResource {
115    pub url: String,
116    pub resource_type: ResourceType,
117    pub content: String,
118    pub content_type: Option<String>,
119}
120
121/// Fetch all resources concurrently.
122///
123/// Respects `block_types` — blocked resources are skipped.
124/// Caps concurrent requests at `max_concurrent` (default 6).
125/// Individual resource timeout: 5s. Total timeout: 15s.
126pub async fn fetch_resources(
127    urls: Vec<ResourceUrl>,
128    block_types: &HashSet<ResourceType>,
129    max_concurrent: usize,
130) -> Vec<LoadedResource> {
131    let filtered = filter_by_block_types(urls, block_types);
132    if filtered.is_empty() {
133        return Vec::new();
134    }
135
136    let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(max_concurrent));
137    let client = hpx::Client::new();
138
139    let total = tokio::time::timeout(std::time::Duration::from_secs(15), async {
140        let mut set = tokio::task::JoinSet::new();
141
142        for resource in filtered {
143            let sem = semaphore.clone();
144            let client = client.clone();
145            set.spawn(async move {
146                let _permit = match sem.acquire().await {
147                    Ok(p) => p,
148                    Err(_) => return None,
149                };
150                let result = tokio::time::timeout(
151                    std::time::Duration::from_secs(5),
152                    client.get(&resource.url).send(),
153                )
154                .await;
155
156                match result {
157                    Ok(Ok(resp)) => {
158                        let content_type = resp
159                            .headers()
160                            .get("content-type")
161                            .and_then(|v| v.to_str().ok())
162                            .map(|s| s.to_string());
163                        let url = resp.uri().to_string();
164                        match resp.text().await {
165                            Ok(text) => Some(LoadedResource {
166                                url,
167                                resource_type: resource.resource_type,
168                                content: text,
169                                content_type,
170                            }),
171                            Err(_) => None,
172                        }
173                    }
174                    _ => None,
175                }
176            });
177        }
178
179        let mut results = Vec::new();
180        while let Some(res) = set.join_next().await {
181            if let Ok(Some(loaded)) = res {
182                results.push(loaded);
183            }
184        }
185        results
186    })
187    .await;
188
189    total.unwrap_or_default()
190}
191
192/// Filter resources by blocked types.
193pub fn filter_by_block_types(
194    resources: Vec<ResourceUrl>,
195    block_types: &HashSet<ResourceType>,
196) -> Vec<ResourceUrl> {
197    resources
198        .into_iter()
199        .filter(|r| !block_types.contains(&r.resource_type))
200        .collect()
201}
202
203#[cfg(test)]
204mod tests {
205    use super::*;
206    use crate::html_parser::parse_html;
207
208    #[test]
209    fn extract_stylesheet() {
210        let dom = parse_html(
211            r#"<html><head><link rel="stylesheet" href="/style.css"></head><body></body></html>"#,
212        );
213        let urls = extract_resource_urls(&dom);
214        assert_eq!(urls.len(), 1);
215        assert_eq!(urls[0].resource_type, ResourceType::Stylesheet);
216        assert_eq!(urls[0].url, "/style.css");
217    }
218
219    #[test]
220    fn extract_script() {
221        let dom =
222            parse_html(r#"<html><head><script src="/app.js"></script></head><body></body></html>"#);
223        let urls = extract_resource_urls(&dom);
224        assert_eq!(urls.len(), 1);
225        assert_eq!(urls[0].resource_type, ResourceType::Script);
226        assert_eq!(urls[0].url, "/app.js");
227    }
228
229    #[test]
230    fn extract_img() {
231        let dom = parse_html(r#"<html><body><img src="/logo.png"></body></html>"#);
232        let urls = extract_resource_urls(&dom);
233        assert_eq!(urls.len(), 1);
234        assert_eq!(urls[0].resource_type, ResourceType::Image);
235        assert_eq!(urls[0].url, "/logo.png");
236    }
237
238    #[test]
239    fn extract_video_and_audio() {
240        let dom = parse_html(
241            r#"<html><body><video src="/v.mp4"></video><audio src="/a.mp3"></audio></body></html>"#,
242        );
243        let urls = extract_resource_urls(&dom);
244        assert_eq!(urls.len(), 2);
245        assert!(urls.iter().all(|r| r.resource_type == ResourceType::Media));
246    }
247
248    #[test]
249    fn extract_preload_font() {
250        let dom = parse_html(
251            r#"<html><head><link rel="preload" href="/font.woff2" as="font" crossorigin></head><body></body></html>"#,
252        );
253        let urls = extract_resource_urls(&dom);
254        assert_eq!(urls.len(), 1);
255        assert_eq!(urls[0].resource_type, ResourceType::Font);
256        assert_eq!(urls[0].url, "/font.woff2");
257    }
258
259    #[test]
260    fn inline_script_ignored() {
261        let dom = parse_html(r#"<html><head><script>alert(1)</script></head><body></body></html>"#);
262        let urls = extract_resource_urls(&dom);
263        assert!(urls.is_empty());
264    }
265
266    #[test]
267    fn all_resource_types() {
268        let dom = parse_html(
269            r#"<html>
270            <head>
271                <link rel="stylesheet" href="/style.css">
272                <script src="/app.js"></script>
273                <link rel="preload" href="/font.woff2" as="font">
274            </head>
275            <body>
276                <img src="/photo.jpg">
277                <video src="/clip.mp4"></video>
278            </body>
279            </html>"#,
280        );
281        let urls = extract_resource_urls(&dom);
282        assert_eq!(urls.len(), 5);
283        let types: HashSet<ResourceType> = urls.iter().map(|r| r.resource_type).collect();
284        assert!(types.contains(&ResourceType::Stylesheet));
285        assert!(types.contains(&ResourceType::Script));
286        assert!(types.contains(&ResourceType::Image));
287        assert!(types.contains(&ResourceType::Font));
288        assert!(types.contains(&ResourceType::Media));
289    }
290
291    #[test]
292    fn dedup_same_url() {
293        let dom = parse_html(
294            r#"<html><head>
295                <link rel="stylesheet" href="/style.css">
296                <link rel="stylesheet" href="/style.css">
297            </head><body></body></html>"#,
298        );
299        let urls = extract_resource_urls(&dom);
300        assert_eq!(urls.len(), 1);
301    }
302
303    #[test]
304    fn filter_blocks_stylesheets() {
305        let dom = parse_html(
306            r#"<html><head>
307                <link rel="stylesheet" href="/style.css">
308                <script src="/app.js"></script>
309            </head><body><img src="/logo.png"></body></html>"#,
310        );
311        let urls = extract_resource_urls(&dom);
312        let mut block = HashSet::new();
313        block.insert(ResourceType::Stylesheet);
314        let filtered = filter_by_block_types(urls, &block);
315        assert_eq!(filtered.len(), 2);
316        assert!(
317            filtered
318                .iter()
319                .all(|r| r.resource_type != ResourceType::Stylesheet)
320        );
321    }
322
323    #[test]
324    fn filter_empty_block_returns_all() {
325        let dom = parse_html(
326            r#"<html><head><link rel="stylesheet" href="/s.css"></head><body><img src="/i.png"></body></html>"#,
327        );
328        let urls = extract_resource_urls(&dom);
329        let block = HashSet::new();
330        let filtered = filter_by_block_types(urls.clone(), &block);
331        assert_eq!(filtered.len(), urls.len());
332    }
333
334    #[test]
335    fn filter_all_blocks_returns_empty() {
336        let dom = parse_html(
337            r#"<html><head>
338                <link rel="stylesheet" href="/style.css">
339                <script src="/app.js"></script>
340            </head><body><img src="/logo.png"></body></html>"#,
341        );
342        let urls = extract_resource_urls(&dom);
343        let mut block = HashSet::new();
344        block.insert(ResourceType::Stylesheet);
345        block.insert(ResourceType::Script);
346        block.insert(ResourceType::Image);
347        block.insert(ResourceType::Font);
348        block.insert(ResourceType::Media);
349        let filtered = filter_by_block_types(urls, &block);
350        assert!(filtered.is_empty());
351    }
352
353    #[test]
354    fn filter_idempotent() {
355        let dom = parse_html(
356            r#"<html><head>
357                <link rel="stylesheet" href="/style.css">
358                <script src="/app.js"></script>
359            </head><body><img src="/logo.png"></body></html>"#,
360        );
361        let urls = extract_resource_urls(&dom);
362        let mut block = HashSet::new();
363        block.insert(ResourceType::Stylesheet);
364        let once = filter_by_block_types(urls.clone(), &block);
365        let twice = filter_by_block_types(once.clone(), &block);
366        assert_eq!(once, twice);
367    }
368
369    #[test]
370    fn block_images_from_html() {
371        let dom =
372            parse_html(r#"<html><body><img src="/photo.jpg"><img src="/icon.png"></body></html>"#);
373        let urls = extract_resource_urls(&dom);
374        assert_eq!(urls.len(), 2);
375        assert!(urls.iter().all(|r| r.resource_type == ResourceType::Image));
376        let mut block = HashSet::new();
377        block.insert(ResourceType::Image);
378        let filtered = filter_by_block_types(urls, &block);
379        assert!(filtered.is_empty());
380    }
381
382    #[test]
383    fn no_resources() {
384        let dom = parse_html(r#"<html><body><p>Hello</p></body></html>"#);
385        let urls = extract_resource_urls(&dom);
386        assert!(urls.is_empty());
387    }
388
389    #[tokio::test]
390    async fn fetch_resources_concurrent() {
391        use tokio::io::AsyncWriteExt;
392
393        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
394        let addr = listener.local_addr().unwrap();
395        let base = format!("http://{addr}");
396
397        let resources = vec![
398            ("/style.css", "text/css", "body{}"),
399            ("/app.js", "application/javascript", "alert(1)"),
400            ("/logo.png", "image/png", "PNGDATA"),
401        ];
402
403        let server_resources = resources.clone();
404        let server = tokio::spawn(async move {
405            for (path, ct, body) in &server_resources {
406                let (mut stream, _) = listener.accept().await.unwrap();
407                let response = format!(
408                    "HTTP/1.1 200 OK\r\nContent-Type: {ct}\r\nContent-Length: {}\r\n\r\n{body}",
409                    body.len()
410                );
411                stream.write_all(response.as_bytes()).await.unwrap();
412            }
413        });
414
415        let urls: Vec<ResourceUrl> = resources
416            .iter()
417            .map(|(path, _, _)| ResourceUrl {
418                resource_type: if path.ends_with(".css") {
419                    ResourceType::Stylesheet
420                } else if path.ends_with(".js") {
421                    ResourceType::Script
422                } else {
423                    ResourceType::Image
424                },
425                url: format!("{base}{path}"),
426            })
427            .collect();
428
429        let loaded = fetch_resources(urls, &HashSet::new(), 6).await;
430        server.await.unwrap();
431
432        assert_eq!(loaded.len(), 3);
433        let mut contents: Vec<&str> = loaded.iter().map(|r| r.content.as_str()).collect();
434        contents.sort();
435        assert_eq!(contents, vec!["PNGDATA", "alert(1)", "body{}"]);
436    }
437
438    #[tokio::test]
439    async fn fetch_resources_respects_block_types() {
440        use tokio::io::AsyncWriteExt;
441
442        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
443        let addr = listener.local_addr().unwrap();
444        let base = format!("http://{addr}");
445
446        let server = tokio::spawn(async move {
447            let (mut stream, _) = listener.accept().await.unwrap();
448            let body = "body{}";
449            let response = format!(
450                "HTTP/1.1 200 OK\r\nContent-Type: text/css\r\nContent-Length: {}\r\n\r\n{body}",
451                body.len()
452            );
453            stream.write_all(response.as_bytes()).await.unwrap();
454        });
455
456        let urls = vec![ResourceUrl {
457            resource_type: ResourceType::Stylesheet,
458            url: format!("{base}/style.css"),
459        }];
460
461        let mut block = HashSet::new();
462        block.insert(ResourceType::Stylesheet);
463        let loaded = fetch_resources(urls, &block, 6).await;
464
465        assert!(loaded.is_empty());
466        // server never got a connection since it was blocked
467    }
468
469    #[tokio::test]
470    async fn fetch_resources_empty_input() {
471        let loaded = fetch_resources(vec![], &HashSet::new(), 6).await;
472        assert!(loaded.is_empty());
473    }
474}
475
476#[cfg(test)]
477#[cfg(feature = "proptest")]
478mod proptests {
479    use proptest::prelude::*;
480
481    use super::*;
482    use crate::html_parser::parse_html;
483
484    fn url_strategy() -> impl Strategy<Value = String> {
485        prop::collection::vec("[a-zA-Z0-9/._-]", 1..20).prop_map(|chars| chars.join(""))
486    }
487
488    fn resource_tag_strategy() -> impl Strategy<Value = String> {
489        prop_oneof![
490            // link stylesheet
491            url_strategy().prop_map(|href| format!(r#"<link rel="stylesheet" href="{}">"#, href)),
492            // script src
493            url_strategy().prop_map(|src| format!(r#"<script src="{}"></script>"#, src)),
494            // img src
495            url_strategy().prop_map(|src| format!(r#"<img src="{}">"#, src)),
496            // video src
497            url_strategy().prop_map(|src| format!(r#"<video src="{}"></video>"#, src)),
498            // audio src
499            url_strategy().prop_map(|src| format!(r#"<audio src="{}"></audio>"#, src)),
500        ]
501    }
502
503    fn html_with_resources_strategy() -> impl Strategy<Value = String> {
504        prop::collection::vec(resource_tag_strategy(), 0..10).prop_map(|tags| {
505            format!(
506                r#"<html><head>{}</head><body>{}</body></html>"#,
507                tags[..tags.len().min(tags.len() / 2)].join(""),
508                tags[tags.len().min(tags.len() / 2)..].join("")
509            )
510        })
511    }
512
513    proptest! {
514        #[test]
515        fn extracted_urls_are_nonempty_strings(html in html_with_resources_strategy()) {
516            let dom = parse_html(&html);
517            let urls = extract_resource_urls(&dom);
518            for r in &urls {
519                prop_assert!(!r.url.is_empty());
520            }
521        }
522
523        #[test]
524        fn block_filtering_removes_blocked_types(html in html_with_resources_strategy()) {
525            let dom = parse_html(&html);
526            let urls = extract_resource_urls(&dom);
527            let mut block = HashSet::new();
528            block.insert(ResourceType::Stylesheet);
529            let filtered = filter_by_block_types(urls, &block);
530            for r in &filtered {
531                prop_assert_ne!(r.resource_type, ResourceType::Stylesheet);
532            }
533        }
534
535        #[test]
536        fn block_filtering_is_idempotent(html in html_with_resources_strategy()) {
537            let dom = parse_html(&html);
538            let urls = extract_resource_urls(&dom);
539            let mut block = HashSet::new();
540            block.insert(ResourceType::Script);
541            let once = filter_by_block_types(urls.clone(), &block);
542            let twice = filter_by_block_types(once.clone(), &block);
543            prop_assert_eq!(once, twice);
544        }
545    }
546}
hpx_browser/resource_loader.rs

hpx_browser/
resource_loader.rs
