millipede-core 0.1.1

Core primitives for the Millipede web crawler: request model, storage traits, events, errors, configuration.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

use super::{CrawlerHandle, CrawlerKind, RequestEnv, RequestOutcome};
use crate::{errors::CrawlError, request::Request};
use futures_util::future::BoxFuture;
use std::sync::Arc;

/// The no-fetch crawler kind whose execution hands requests directly to handlers.
#[derive(Debug, Clone, Copy, Default)]
pub struct BasicKind;

/// The handler context for [`BasicKind`].
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct BasicContext {
    /// The request being handled.
    pub request: Arc<Request>,
    /// A weak back-reference to the running crawler.
    pub crawler: CrawlerHandle,
}

impl crate::router::HasRequest for BasicContext {
    fn request(&self) -> &Request {
        &self.request
    }
}

impl CrawlerKind for BasicKind {
    type Context = BasicContext;

    fn execute<'a>(
        &'a self,
        env: RequestEnv<'a>,
    ) -> BoxFuture<'a, Result<BasicContext, CrawlError>> {
        Box::pin(async move {
            Ok(BasicContext {
                request: env.request,
                crawler: env.crawler,
            })
        })
    }

    fn cleanup(
        &self,
        _outcome: RequestOutcome<BasicContext>,
    ) -> BoxFuture<'_, Result<(), CrawlError>> {
        Box::pin(async { Ok(()) })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{
        config::Configuration,
        crawler::{CrawlerEnv, RequestPrep},
        router::HasRequest,
    };

    #[tokio::test]
    async fn basic_kind_execute_is_identity() {
        let config = Arc::new(Configuration::default());
        let shared = crate::crawler::tests::shared();
        let crawler_env = CrawlerEnv {
            shared: shared.clone(),
            config,
            storage: None,
            kvs: None,
        };
        let request = Arc::new(Request::get("https://example.com/").build().unwrap());
        let kind = BasicKind;

        assert!(kind.start(&crawler_env).await.is_ok());
        let mut prep = RequestPrep {
            request: (*request).clone(),
        };
        assert!(kind.before_request(&mut prep).await.is_ok());
        let mut context = kind
            .execute(RequestEnv {
                request: request.clone(),
                crawler: crawler_env.handle(),
                events: crawler_env.events(),
                overrides: Default::default(),
            })
            .await
            .unwrap();
        assert!(Arc::ptr_eq(&context.request, &request));
        assert!(std::ptr::eq(context.request(), request.as_ref()));
        assert!(kind.after_success(&mut context).await.is_ok());
        assert!(kind.stop(&crawler_env).await.is_ok());
    }
}