fake_useragent/
lib.rs

1extern crate reqwest;
2extern crate select;
3
4pub use user_agent_string::{
5    browser::Browsers,
6};
7
8mod user_agent_string;
9
10// --- custom ---
11use user_agent_string::UserAgentString;
12
13pub struct UserAgents(Vec<String>);
14
15impl UserAgents {
16    pub fn new() -> UserAgents {
17        // --- std ---
18        use std::{
19            env::current_dir,
20            fs::File,
21            io::Read,
22            path::Path,
23        };
24
25        let path = format!("{}/user_agents", current_dir().unwrap().to_str().unwrap());
26        let path = Path::new(&path);
27        if path.is_file() {
28            let mut file = File::open(path).unwrap();
29            let mut s = String::new();
30            file.read_to_string(&mut s).unwrap();
31
32            UserAgents(s.lines().map(|user_agent| user_agent.to_owned()).collect())
33        } else { UserAgentsBuilder::new().all_browsers().build() }
34    }
35
36    pub fn from_cache(path: &str) -> UserAgents {
37        // --- std ---
38        use std::{
39            fs::File,
40            io::Read,
41        };
42
43        let mut file = File::open(path).unwrap();
44        let mut s = String::new();
45        file.read_to_string(&mut s).unwrap();
46
47        UserAgents(s.lines().map(|user_agent| user_agent.to_owned()).collect())
48    }
49
50    pub fn random(&self) -> &str {
51        // --- external ---
52        use rand::{
53            thread_rng,
54            Rng,
55        };
56
57        &self.0[thread_rng().gen_range(0, self.0.len())]
58    }
59}
60
61pub struct UserAgentsBuilder<'a> {
62    cache: bool,
63    thread: u32,
64    dir: String,
65    user_agents: Vec<String>,
66    kinds: [Option<Box<dyn UserAgentString + 'a>>; 13],
67}
68
69macro_rules! set_unset {
70    ($self_:ident, $setter:ident, $unsetter:ident, $i:expr) => {
71        pub fn $setter(mut $self_, browsers: Browsers<'a>) -> Self {
72            $self_.kinds[$i] = Some(Box::new(browsers));
73            $self_
74        }
75
76        pub fn $unsetter(mut $self_) -> Self {
77            $self_.kinds[$i] = None;
78            $self_
79        }
80    };
81}
82
83impl<'a> UserAgentsBuilder<'a> {
84    pub fn new() -> UserAgentsBuilder<'a> {
85        UserAgentsBuilder {
86            cache: true,
87            thread: 20,
88            dir: std::env::current_dir()
89                .unwrap()
90                .to_str()
91                .unwrap()
92                .to_owned(),
93            user_agents: vec![],
94            kinds: [None, None, None, None, None, None, None, None, None, None, None, None, None],
95        }
96    }
97
98    pub fn all_browsers(mut self) -> Self {
99        self.kinds[1] = Some(Box::new(Browsers::new().set_all()));
100        self
101    }
102    set_unset!(self, set_browsers, unset_browsers, 1);
103
104
105    fn get(url: &str) -> String {
106        let client = reqwest::ClientBuilder::new()
107            .danger_accept_invalid_certs(true)
108            .danger_accept_invalid_hostnames(true)
109            .gzip(true)
110            .build()
111            .unwrap();
112
113        loop {
114            match client.get(url).send() {
115                Ok(mut resp) => match resp.text() {
116                    Ok(html) => return html,
117                    Err(e) => println!("{:?}", e)
118                }
119                Err(e) => println!("{:?}", e)
120            }
121        }
122    }
123
124    fn parse(html: String) -> Vec<String> {
125        // --- external ---
126        use select::{
127            document::Document,
128            predicate::{Attr, Name, Predicate},
129        };
130
131        let mut user_agents = vec![];
132        let document = Document::from(html.as_str());
133        for node in document.find(Attr("id", "liste").descendant(Name("ul"))) {
134            for user_agent in node.find(Name("li").descendant(Name("a"))) {
135                user_agents.push(user_agent.text());
136            }
137        }
138
139        user_agents
140    }
141
142    pub fn cache(mut self, cache: bool) -> Self {
143        self.cache = cache;
144        self
145    }
146
147    pub fn thread(mut self, num: u32) -> Self {
148        if num >= 382 { self.thread = 382; } else if num > 0 { self.thread = num; }
149        self
150    }
151
152    pub fn dir(mut self, path: &str) -> Self {
153        self.dir = path.trim_end_matches('/').to_owned();
154        self
155    }
156
157    fn store(&self) {
158        // --- std ---
159        use std::{
160            fs::File,
161            io::Write,
162        };
163
164        let mut file = File::create(format!("{}/user_agents", self.dir)).unwrap();
165        file.write_all(self.user_agents.join("\n").as_bytes()).unwrap();
166        file.sync_all().unwrap();
167    }
168
169    pub fn build(mut self) -> UserAgents {
170        for user_agent_string in self.kinds.iter() {
171            if let Some(user_agent_string) = user_agent_string {
172                user_agent_string.fetch(self.thread, &mut self.user_agents);
173            }
174        }
175
176        if self.cache { self.store(); }
177
178        UserAgents(self.user_agents)
179    }
180}
fake_useragent/lib.rs

fake_useragent/
lib.rs
