millipede_core/crawler/
basic.rs

1use super::{CrawlerHandle, CrawlerKind, RequestEnv, RequestOutcome};
2use crate::{errors::CrawlError, request::Request};
3use futures_util::future::BoxFuture;
4use std::sync::Arc;
5
6/// The no-fetch crawler kind whose execution hands requests directly to handlers.
7#[derive(Debug, Clone, Copy, Default)]
8pub struct BasicKind;
9
10/// The handler context for [`BasicKind`].
11#[derive(Debug, Clone)]
12#[non_exhaustive]
13pub struct BasicContext {
14    /// The request being handled.
15    pub request: Arc<Request>,
16    /// A weak back-reference to the running crawler.
17    pub crawler: CrawlerHandle,
18}
19
20impl crate::router::HasRequest for BasicContext {
21    fn request(&self) -> &Request {
22        &self.request
23    }
24}
25
26impl CrawlerKind for BasicKind {
27    type Context = BasicContext;
28
29    fn execute<'a>(
30        &'a self,
31        env: RequestEnv<'a>,
32    ) -> BoxFuture<'a, Result<BasicContext, CrawlError>> {
33        Box::pin(async move {
34            Ok(BasicContext {
35                request: env.request,
36                crawler: env.crawler,
37            })
38        })
39    }
40
41    fn cleanup(
42        &self,
43        _outcome: RequestOutcome<BasicContext>,
44    ) -> BoxFuture<'_, Result<(), CrawlError>> {
45        Box::pin(async { Ok(()) })
46    }
47}
48
49#[cfg(test)]
50mod tests {
51    use super::*;
52    use crate::{
53        config::Configuration,
54        crawler::{CrawlerEnv, RequestPrep},
55        router::HasRequest,
56    };
57
58    #[tokio::test]
59    async fn basic_kind_execute_is_identity() {
60        let config = Arc::new(Configuration::default());
61        let shared = crate::crawler::tests::shared();
62        let crawler_env = CrawlerEnv {
63            shared: shared.clone(),
64            config,
65            storage: None,
66            kvs: None,
67        };
68        let request = Arc::new(Request::get("https://example.com/").build().unwrap());
69        let kind = BasicKind;
70
71        assert!(kind.start(&crawler_env).await.is_ok());
72        let mut prep = RequestPrep {
73            request: (*request).clone(),
74        };
75        assert!(kind.before_request(&mut prep).await.is_ok());
76        let mut context = kind
77            .execute(RequestEnv {
78                request: request.clone(),
79                crawler: crawler_env.handle(),
80                events: crawler_env.events(),
81                overrides: Default::default(),
82            })
83            .await
84            .unwrap();
85        assert!(Arc::ptr_eq(&context.request, &request));
86        assert!(std::ptr::eq(context.request(), request.as_ref()));
87        assert!(kind.after_success(&mut context).await.is_ok());
88        assert!(kind.stop(&crawler_env).await.is_ok());
89    }
90}
millipede_core/crawler/basic.rs

millipede_core/crawler/
basic.rs
