spire-core 0.9.0

Core types and traits for Spire web scraping framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use ::http::Uri;

use crate::crawler::Tag;
use crate::frontier::seed::Seed;

/// A link to follow with [`Queue::follow`](crate::frontier::queue::Queue::follow).
///
/// A link is either an `href`, absolute or relative to the page it was found
/// on, or a complete [`Seed`]. Strings, URIs and seeds convert into links.
///
/// ```
/// use spire_core::frontier::queue::Link;
///
/// let next = Link::from("/page/2");
/// let detail = Link::from("../items/42").with_tag("item");
/// let next = Link::from("/page/3").with_priority(10);
/// ```
#[derive(Debug, Clone)]
pub struct Link {
    pub(crate) target: Target,
    pub(crate) tag: Option<Tag>,
    pub(crate) priority: Option<i32>,
}

#[derive(Debug, Clone)]
pub(crate) enum Target {
    Href(String),
    Seed(Box<Seed>),
}

impl Link {
    /// Creates a link from an `href`, absolute or relative.
    #[must_use]
    pub fn new(href: impl Into<String>) -> Self {
        Self {
            target: Target::Href(href.into()),
            tag: None,
            priority: None,
        }
    }

    /// Routes the followed request to the handler for `tag`.
    #[must_use]
    pub fn with_tag(mut self, tag: impl Into<Tag>) -> Self {
        self.tag = Some(tag.into());
        self
    }

    /// Sets the followed request's priority, e.g. scored from the page the
    /// link was found on; frontiers that order by priority crawl higher
    /// priorities first. A [scoring](crate::frontier::FrontierExt::score)
    /// frontier adds its score to it.
    #[must_use]
    pub const fn with_priority(mut self, priority: i32) -> Self {
        self.priority = Some(priority);
        self
    }
}

impl From<&str> for Link {
    fn from(href: &str) -> Self {
        Self::new(href)
    }
}

impl From<String> for Link {
    fn from(href: String) -> Self {
        Self::new(href)
    }
}

impl From<&String> for Link {
    fn from(href: &String) -> Self {
        Self::new(href.as_str())
    }
}

impl From<Uri> for Link {
    fn from(uri: Uri) -> Self {
        Self::new(uri.to_string())
    }
}

impl From<Seed> for Link {
    fn from(seed: Seed) -> Self {
        Self {
            target: Target::Seed(Box::new(seed)),
            tag: None,
            priority: None,
        }
    }
}