spire-core 0.8.0

Core types and traits for Spire web scraping framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use std::time::Duration;

use super::Tags;

/// How often the pages of each tag are crawled again, for a crawl that keeps
/// its data fresh, see [`FrontierExt::revisit`].
///
/// With the `macros` feature, `#[derive(Revisit)]` implements it from a
/// `#[revisit("1d")]` attribute on each variant to revisit.
///
/// ```
/// use std::time::Duration;
///
/// use spire_core::crawler::{Revisit, Tag, Tags};
///
/// #[derive(Clone, Copy)]
/// enum Page {
///     Listing,
///     About,
/// }
///
/// impl From<Page> for Tag {
///     fn from(page: Page) -> Self {
///         Tag::from_static(match page {
///             Page::Listing => "listing",
///             Page::About => "about",
///         })
///     }
/// }
///
/// impl Tags for Page {
///     const ALL: &'static [Self] = &[Self::Listing, Self::About];
/// }
///
/// impl Revisit for Page {
///     fn interval(self) -> Option<Duration> {
///         match self {
///             Page::Listing => Some(Duration::from_hours(1)),
///             Page::About => None,
///         }
///     }
/// }
/// ```
///
/// [`FrontierExt::revisit`]: crate::frontier::FrontierExt::revisit
pub trait Revisit: Tags {
    /// How long after a page of this tag is crawled it is crawled again;
    /// `None` crawls it once.
    fn interval(self) -> Option<Duration>;
}