spire_core/identity/
mod.rs

1//! Identities: who a request appears to come from.
2//!
3//! A request reveals its sender in several places at once: its IP, through
4//! a proxy; its user agent and the headers that go with it; and its cookies.
5//! An [`Identity`] keeps them together, one visitor, so they always agree:
6//! one IP does not send many user agents, and a session cookie set through
7//! one proxy is not sent through another.
8//!
9//! [`Identities`] are defined once, on the
10//! [`Crawler`](crate::crawler::Crawler::with_identities), which picks one for every
11//! request, applies its headers, and connects as it, see
12//! [`Backend::connect_as`](crate::backend::Backend::connect_as). Identities that
13//! keep failing are set aside, or replaced by new ones.
14//!
15//! ```
16//! use spire_core::Result;
17//! use spire_core::http::http::HeaderValue;
18//! use spire_core::http::http::header::USER_AGENT;
19//! use spire_core::identity::{Identities, Identity};
20//!
21//! # fn main() -> Result<()> {
22//! // A rotating gateway: each session id is a different exit IP.
23//! let identities = Identities::from_fn(|n| {
24//!     Ok(Identity::new(format!("session-{n}"))
25//!         .with_proxy(format!("http://user-session-{n}:secret@gate.example:7000"))?
26//!         .with_header(USER_AGENT, HeaderValue::from_static("MyBot/1.0")))
27//! })
28//! .max_active(100)
29//! .sticky_per_host();
30//! # Ok(())
31//! # }
32//! ```
33
34mod identities;
35mod pool;
36mod weak;
37
38use std::fmt;
39use std::hash::{Hash, Hasher};
40use std::sync::Arc;
41
42use ::http::header::IntoHeaderName;
43use ::http::{HeaderMap, HeaderValue, Uri};
44#[cfg(feature = "user-agents")]
45#[cfg_attr(docsrs, doc(cfg(feature = "user-agents")))]
46#[doc(inline)]
47pub use uagent::{Browser, Platform, Profile, UserAgents};
48
49pub use self::identities::Identities;
50pub(crate) use self::pool::{Health, IdentityPool};
51pub use self::weak::WeakIdentity;
52use crate::http::{Request, RequestMeta};
53use crate::{Error, Result};
54
55/// Who a request appears to come from: a name, an optional proxy, and the
56/// headers it sends.
57///
58/// Identities are compared and hashed by name. Cloning is cheap.
59#[derive(Clone)]
60pub struct Identity(Arc<Inner>);
61
62#[derive(Clone)]
63struct Inner {
64    name: Box<str>,
65    proxy: Option<Uri>,
66    headers: HeaderMap,
67    #[cfg(feature = "user-agents")]
68    profile: Option<Profile>,
69}
70
71impl Identity {
72    /// Creates an identity named `name`, with no proxy and no headers.
73    #[must_use]
74    pub fn new(name: impl Into<Box<str>>) -> Self {
75        Self(Arc::new(Inner {
76            name: name.into(),
77            proxy: None,
78            headers: HeaderMap::new(),
79            #[cfg(feature = "user-agents")]
80            profile: None,
81        }))
82    }
83
84    /// Fails if requests should go through a proxy: for a backend that
85    /// cannot route through one, so it never sends them from the crawler's
86    /// own IP instead.
87    ///
88    /// # Errors
89    ///
90    /// Returns an [`Invalid`](crate::ErrorKind::Invalid) error if the
91    /// identity has a proxy.
92    pub fn require_no_proxy(&self) -> Result<()> {
93        match self.proxy() {
94            Some(_) => Err(Error::invalid(format!(
95                "identity `{}` has a proxy, which this backend cannot send through",
96                self.name()
97            ))),
98            None => Ok(()),
99        }
100    }
101
102    /// Sends requests through `proxy`, an `http`, `https` or `socks5` URL,
103    /// with credentials if it needs them.
104    ///
105    /// # Errors
106    ///
107    /// Returns an [`Invalid`](crate::ErrorKind::Invalid) error if `proxy`
108    /// is malformed or not absolute.
109    pub fn with_proxy<U>(mut self, proxy: U) -> Result<Self>
110    where
111        U: TryInto<Uri>,
112        U::Error: Into<Error>,
113    {
114        let proxy: Uri = proxy.try_into().map_err(Into::into)?;
115        if proxy.scheme().is_none() || proxy.authority().is_none() {
116            let message = format!("proxy of identity `{}` is not absolute", self.name());
117            return Err(Error::invalid(message));
118        }
119        Arc::make_mut(&mut self.0).proxy = Some(proxy);
120        Ok(self)
121    }
122
123    /// Sends `name` with `value` on every request, unless the request sets
124    /// it itself.
125    #[must_use]
126    pub fn with_header(mut self, name: impl IntoHeaderName, value: HeaderValue) -> Self {
127        Arc::make_mut(&mut self.0).headers.insert(name, value);
128        self
129    }
130
131    /// Presents as this browser profile. Identity headers override the
132    /// profile's headers, and request headers override both.
133    ///
134    /// Real browser backends refuse profiles: their own browser supplies
135    /// its user agent and client hints.
136    #[cfg(feature = "user-agents")]
137    #[cfg_attr(docsrs, doc(cfg(feature = "user-agents")))]
138    #[must_use]
139    pub fn with_profile(mut self, profile: Profile) -> Self {
140        Arc::make_mut(&mut self.0).profile = Some(profile);
141        self
142    }
143
144    /// Returns the browser profile, kept separate from identity headers.
145    #[cfg(feature = "user-agents")]
146    #[cfg_attr(docsrs, doc(cfg(feature = "user-agents")))]
147    #[must_use]
148    pub fn profile(&self) -> Option<&Profile> {
149        self.0.profile.as_ref()
150    }
151
152    /// Fails if this identity has a browser profile, for backends whose
153    /// real browser supplies its own user agent and client hints.
154    ///
155    /// # Errors
156    ///
157    /// Returns an [`Invalid`](crate::ErrorKind::Invalid) error if a profile
158    /// is set. Without `user-agents`, always succeeds.
159    pub fn require_no_profile(&self) -> Result<()> {
160        #[cfg(feature = "user-agents")]
161        if self.profile().is_some() {
162            return Err(Error::invalid(format!(
163                "identity `{}` has a user-agent profile, which a real browser cannot use",
164                self.name()
165            )));
166        }
167        Ok(())
168    }
169
170    /// Returns the name.
171    #[must_use]
172    pub fn name(&self) -> &str {
173        &self.0.name
174    }
175
176    /// Returns the proxy, if requests go through one.
177    #[must_use]
178    pub fn proxy(&self) -> Option<&Uri> {
179        self.0.proxy.as_ref()
180    }
181
182    /// Returns the identity headers, excluding its browser profile.
183    #[must_use]
184    pub fn headers(&self) -> &HeaderMap {
185        &self.0.headers
186    }
187
188    /// Returns a weak reference to this identity, which tells whether it is
189    /// still in use: once the crawl retires it and its last request ends,
190    /// it is gone. Backends use it to drop what they keep per identity,
191    /// such as a client or a browser profile.
192    #[must_use]
193    pub fn downgrade(&self) -> WeakIdentity {
194        WeakIdentity(Arc::downgrade(&self.0))
195    }
196
197    /// Sends `request` as this identity: with its headers, unless the
198    /// request sets them, and recorded for
199    /// [`RequestExt::identity`](crate::http::RequestExt::identity).
200    pub(crate) fn apply(&self, request: &mut Request) {
201        let headers = request.headers_mut();
202        for (name, value) in self.headers() {
203            if !headers.contains_key(name) {
204                headers.insert(name.clone(), value.clone());
205            }
206        }
207        #[cfg(feature = "user-agents")]
208        if let Some(profile) = self.profile() {
209            for header in profile.headers() {
210                if !headers.contains_key(header.name) {
211                    // uagent generates validated names and values from its captures.
212                    let name = ::http::HeaderName::from_static(header.name);
213                    let value = HeaderValue::from_str(&header.value)
214                        .expect("uagent profile headers are valid HTTP headers");
215                    headers.insert(name, value);
216                }
217            }
218        }
219        let meta = request
220            .extensions_mut()
221            .get_or_insert_default::<RequestMeta>();
222        meta.identity = Some(self.clone());
223    }
224}
225
226/// Shows the proxy without its credentials.
227impl fmt::Debug for Identity {
228    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
229        let proxy = self.proxy().map(|proxy| {
230            let host = proxy.host().unwrap_or_default();
231            match proxy.port_u16() {
232                Some(port) => format!("{host}:{port}"),
233                None => host.to_owned(),
234            }
235        });
236        f.debug_struct("Identity")
237            .field("name", &self.name())
238            .field("proxy", &proxy)
239            .field("headers", &self.headers().keys().collect::<Vec<_>>())
240            .finish()
241    }
242}
243
244impl PartialEq for Identity {
245    fn eq(&self, other: &Self) -> bool {
246        self.name() == other.name()
247    }
248}
249
250impl Eq for Identity {}
251
252impl Hash for Identity {
253    fn hash<H: Hasher>(&self, state: &mut H) {
254        self.name().hash(state);
255    }
256}
257
258#[cfg(test)]
259mod tests {
260    use ::http::header::USER_AGENT;
261
262    use super::*;
263
264    #[cfg(feature = "user-agents")]
265    #[test]
266    fn replacing_a_profile_preserves_explicit_headers_and_clones() {
267        let chrome = Profile::new(Browser::Chrome, Platform::Windows).unwrap();
268        let firefox = Profile::new(Browser::Firefox, Platform::Linux).unwrap();
269        let original = Identity::new("a")
270            .with_profile(chrome.clone())
271            .with_header(USER_AGENT, HeaderValue::from_static("explicit"));
272        let changed = original.clone().with_profile(firefox.clone());
273        assert_eq!(original.profile(), Some(&chrome));
274        assert_eq!(changed.profile(), Some(&firefox));
275        assert_eq!(changed.headers()[USER_AGENT], "explicit");
276        assert_eq!(changed.headers().len(), 1);
277        assert!(changed.require_no_profile().is_err());
278        assert!(Identity::new("plain").require_no_profile().is_ok());
279    }
280
281    #[test]
282    fn debug_hides_proxy_credentials() {
283        let identity = Identity::new("a")
284            // Fake credentials, for the test.
285            .with_proxy("http://user:secret@proxy.test:8080") // trufflehog:ignore
286            .unwrap()
287            .with_header(USER_AGENT, HeaderValue::from_static("Bot"));
288        let debug = format!("{identity:?}");
289        assert!(debug.contains("proxy.test:8080"), "{debug}");
290        assert!(!debug.contains("secret"), "{debug}");
291        assert_eq!(identity.proxy().unwrap().host(), Some("proxy.test"));
292    }
293
294    #[test]
295    fn only_proxied_identities_require_a_proxy() {
296        assert!(Identity::new("a").require_no_proxy().is_ok());
297        let proxied = Identity::new("a")
298            .with_proxy("http://proxy.test:8080")
299            .unwrap();
300        assert!(proxied.require_no_proxy().is_err());
301    }
302
303    #[test]
304    fn rejects_relative_proxies() {
305        assert!(Identity::new("a").with_proxy("/proxy").is_err());
306        assert!(Identity::new("a").with_proxy("not a url").is_err());
307    }
308}
spire_core/identity/mod.rs

spire_core/identity/
mod.rs
