spire_core/identity/
identities.rs

1//! [`Identities`]: where a crawl's identities come from, and how they are
2//! picked and taken out of use.
3
4use std::fmt;
5use std::sync::Arc;
6use std::time::Duration;
7
8use super::Identity;
9use crate::Result;
10
11/// Builds identity `n`, for [`Identities::from_fn`].
12pub(super) type Generate = Arc<dyn Fn(u64) -> Result<Identity> + Send + Sync>;
13
14/// Where a crawl's identities come from.
15#[derive(Clone)]
16pub(super) enum Source {
17    /// A fixed list.
18    Fixed(Vec<Identity>),
19    /// Built on demand, identity `n` by a function.
20    Generate(Generate),
21}
22
23/// How an identity is picked for a request.
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub(super) enum Choice {
26    InTurn,
27    Random,
28    StickyPerHost,
29}
30
31/// The identities of a crawl, and how they are picked, set aside and replaced.
32///
33/// A request pinned with
34/// [`Seed::with_identity`](crate::frontier::seed::Seed::with_identity) goes
35/// out as that identity; any other gets one picked [in turn](Self::in_turn),
36/// [at random](Self::at_random), or [per host](Self::sticky_per_host).
37///
38/// An identity whose requests fail [`max_failures`](Self::max_failures)
39/// times in a row, through connection failures, timeouts, or `403`, `407`
40/// or `429` answers, is taken out of use: a generated one is retired and
41/// replaced by the next, a listed one set aside for a while, longer each
42/// time.
43#[derive(Clone)]
44pub struct Identities {
45    pub(super) source: Source,
46    pub(super) max_active: usize,
47    pub(super) choice: Choice,
48    pub(super) max_requests: Option<u64>,
49    pub(super) max_age: Option<Duration>,
50    pub(super) max_failures: u32,
51    pub(super) rest: Duration,
52}
53
54impl Identities {
55    /// The identities generated at once by default, see
56    /// [`max_active`](Self::max_active).
57    pub const DEFAULT_MAX_ACTIVE: usize = 16;
58
59    /// Uses a fixed list of identities.
60    ///
61    /// # Panics
62    ///
63    /// Panics if `identities` is empty or two share a name.
64    #[must_use]
65    pub fn new(identities: impl IntoIterator<Item = Identity>) -> Self {
66        let identities: Vec<Identity> = identities.into_iter().collect();
67        assert!(
68            !identities.is_empty(),
69            "`Identities::new` needs an identity"
70        );
71        for (index, identity) in identities.iter().enumerate() {
72            assert!(
73                !identities[..index].contains(identity),
74                "two identities are named `{}`",
75                identity.name()
76            );
77        }
78        let max_active = identities.len();
79        Self::with_source(Source::Fixed(identities), max_active)
80    }
81
82    /// Builds identities on demand: identity `n`, counting from 0, from
83    /// `generate`, e.g. a gateway's session `n`, or the `n`th proxy of a
84    /// list. Up to [`max_active`](Self::max_active) are in use at once;
85    /// one taken out of use is replaced by the next `n`.
86    ///
87    /// Names should follow from `n`, so a resumed crawl builds the same
88    /// identities again. A failure to build one fails the request that
89    /// needed it.
90    #[must_use]
91    pub fn from_fn<F>(generate: F) -> Self
92    where
93        F: Fn(u64) -> Result<Identity> + Send + Sync + 'static,
94    {
95        Self::with_source(
96            Source::Generate(Arc::new(generate)),
97            Self::DEFAULT_MAX_ACTIVE,
98        )
99    }
100
101    const fn with_source(source: Source, max_active: usize) -> Self {
102        Self {
103            source,
104            max_active,
105            choice: Choice::InTurn,
106            max_requests: None,
107            max_age: None,
108            max_failures: 3,
109            rest: Duration::from_mins(1),
110        }
111    }
112
113    /// Generates at most `n` identities at once, created as requests need
114    /// them, so a thousand cost nothing until used. Ignored for a fixed
115    /// list.
116    ///
117    /// # Panics
118    ///
119    /// Panics if `n` is zero.
120    #[must_use]
121    pub fn max_active(mut self, n: usize) -> Self {
122        assert!(n > 0, "`Identities::max_active` must be above zero");
123        if matches!(self.source, Source::Generate(_)) {
124            self.max_active = n;
125        }
126        self
127    }
128
129    /// Picks identities in turn, one request each. This is the default.
130    #[must_use]
131    pub const fn in_turn(mut self) -> Self {
132        self.choice = Choice::InTurn;
133        self
134    }
135
136    /// Picks an identity at random for every request.
137    #[must_use]
138    pub const fn at_random(mut self) -> Self {
139        self.choice = Choice::Random;
140        self
141    }
142
143    /// Keeps each host on one identity, as one visitor would be, for as
144    /// long as that identity stays in use; a new host gets a new identity
145    /// while fewer than [`max_active`](Self::max_active) are.
146    #[must_use]
147    pub const fn sticky_per_host(mut self) -> Self {
148        self.choice = Choice::StickyPerHost;
149        self
150    }
151
152    /// Retires a generated identity after `n` requests, replacing it with
153    /// the next.
154    #[must_use]
155    pub const fn retire_after_requests(mut self, n: u64) -> Self {
156        self.max_requests = Some(n);
157        self
158    }
159
160    /// Retires a generated identity once it is `age` old, replacing it with
161    /// the next.
162    #[must_use]
163    pub const fn retire_after(mut self, age: Duration) -> Self {
164        self.max_age = Some(age);
165        self
166    }
167
168    /// Takes an identity out of use after `n` failures in a row, 3 by
169    /// default.
170    ///
171    /// # Panics
172    ///
173    /// Panics if `n` is zero.
174    #[must_use]
175    pub fn max_failures(mut self, n: u32) -> Self {
176        assert!(n > 0, "`Identities::max_failures` must be above zero");
177        self.max_failures = n;
178        self
179    }
180
181    /// Sets a failing listed identity aside for `duration` the first time,
182    /// one minute by default, and twice as long each time after.
183    #[must_use]
184    pub const fn rest(mut self, duration: Duration) -> Self {
185        self.rest = duration;
186        self
187    }
188}
189
190impl fmt::Debug for Identities {
191    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
192        let source = match &self.source {
193            Source::Fixed(identities) => format!("{} listed", identities.len()),
194            Source::Generate(_) => "generated".to_owned(),
195        };
196        f.debug_struct("Identities")
197            .field("source", &source)
198            .field("max_active", &self.max_active)
199            .field("choice", &self.choice)
200            .field("max_requests", &self.max_requests)
201            .field("max_age", &self.max_age)
202            .field("max_failures", &self.max_failures)
203            .field("rest", &self.rest)
204            .finish()
205    }
206}
207
208#[cfg(test)]
209mod tests {
210    use super::*;
211
212    #[test]
213    #[should_panic(expected = "two identities are named `a`")]
214    fn listed_names_are_unique() {
215        let _ = Identities::new([Identity::new("a"), Identity::new("a")]);
216    }
217}
spire_core/identity/identities.rs

spire_core/identity/
identities.rs
