webpage 0.1.2

Small library to fetch info about a web page
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

use curl::easy::Easy;
use std::time::Duration;

#[derive(Debug)]
pub struct HTTP {
    pub ip: String,
    pub transfer_time: Duration,
    pub redirect_count: u32,
    pub content_type: String,
    pub response_code: u32,
    pub headers: Vec<String>,
    pub url: String, // effective url
    pub body: String,
}

impl HTTP {
    pub fn fetch(url: &str) -> Self {
        let mut handle = Easy::new();

        // configure
        handle.timeout(Duration::from_secs(10)).unwrap();
        handle.follow_location(true).unwrap();
        handle.max_redirections(5).unwrap();
        handle.useragent("Webpage - rust crate").unwrap();

        handle.url(url).unwrap();

        let mut headers = Vec::new();
        let mut body = Vec::new();
        {
            let mut transfer = handle.transfer();
            transfer.header_function(|new_data| {
                let header = String::from_utf8_lossy(new_data).into_owned()
                    .trim().to_string();

                // clear list on redirects
                if header.starts_with("HTTP/") {
                    headers = Vec::new();
                }

                if header.len() > 0 {
                    headers.push(header);
                }

                true
            }).unwrap();

            transfer.write_function(|new_data| {
                body.extend_from_slice(new_data);
                Ok(new_data.len())
            }).unwrap();

            transfer.perform().unwrap();
        }

        let body = String::from_utf8_lossy(&body).into_owned();

        HTTP {
            ip: handle.primary_ip().unwrap().unwrap().to_string(),
            transfer_time: handle.total_time().unwrap(),
            redirect_count: handle.redirect_count().unwrap(),
            content_type: handle.content_type().unwrap().unwrap().to_string(),
            response_code: handle.response_code().unwrap(),
            url: handle.effective_url().unwrap().unwrap().to_string(),

            headers,
            body: body,
        }
    }
}