fetcher 0.15.3

Data automation and pipelining framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

//! This module contains the [`Http`] transform that fetches a web page from a link located in a field of the passed [`Entry`]

use reqwest::Client;
use url::Url;

use super::Transform;
use crate::{
	actions::transforms::{
		field::Field,
		result::{TransformResult, TransformedEntry, TransformedMessage},
	},
	entry::Entry,
	error::InvalidUrlError,
	sources::{self, http::HttpError as SourceHttpError, http::Request},
	utils::OptionExt,
};

/// A transform that fetches the page from URL in `from_field` and returns it in [`Entry::raw_contents`]
#[derive(Debug)]
pub struct Http {
	/// The field to get the URL from
	pub from_field: Field,

	/// Reqwest HTTP client
	client: Client,
}

#[expect(missing_docs, reason = "error message is self-documenting")]
#[derive(thiserror::Error, Debug)]
pub enum HttpError {
	#[error("Missing URL in the entry {0} field")]
	MissingUrl(Field),

	#[error("Invalid URL in the entry {0} field")]
	InvalidUrl(Field, #[source] InvalidUrlError),

	#[error(transparent)]
	Other(#[from] crate::sources::http::HttpError),
}

impl Http {
	/// Create a new [`Http`] transform
	///
	/// # Errors
	/// This method fails if TLS couldn't be initialized
	pub fn new(from_field: Field) -> Result<Self, SourceHttpError> {
		let client = sources::http::CLIENT
			.get_or_try_init(|| {
				reqwest::ClientBuilder::new()
					.timeout(std::time::Duration::from_secs(30))
					.build()
					.map_err(SourceHttpError::TlsInitFailed)
			})?
			.clone();

		Ok(Self { from_field, client })
	}
}

impl Transform for Http {
	type Err = HttpError;

	async fn transform_entry(&mut self, entry: Entry) -> Result<Vec<TransformedEntry>, Self::Err> {
		let url: Option<Url> = match self.from_field {
			Field::Title => entry.msg.title.as_deref().try_map(|s| {
				Url::try_from(s).map_err(|e| {
					HttpError::InvalidUrl(self.from_field, InvalidUrlError(e, s.to_owned()))
				})
			})?,
			Field::Body => entry.msg.body.as_deref().try_map(|s| {
				Url::try_from(s).map_err(|e| {
					HttpError::InvalidUrl(self.from_field, InvalidUrlError(e, s.to_owned()))
				})
			})?,
			Field::Link => entry.msg.link.as_deref().try_map(|s| {
				Url::try_from(s).map_err(|e| {
					HttpError::InvalidUrl(self.from_field, InvalidUrlError(e, s.to_owned()))
				})
			})?,
			Field::Id => entry.id.as_ref().try_map(|id| {
				Url::try_from(id.0.as_str()).map_err(|e| {
					HttpError::InvalidUrl(
						self.from_field,
						InvalidUrlError(e, id.0.as_string().clone()),
					)
				})
			})?,
			Field::ReplyTo => entry.reply_to.as_ref().try_map(|id| {
				Url::try_from(id.0.as_str()).map_err(|e| {
					HttpError::InvalidUrl(
						self.from_field,
						InvalidUrlError(e, id.0.as_string().clone()),
					)
				})
			})?,
			Field::RawContents => entry.raw_contents.as_deref().try_map(|s| {
				Url::try_from(s).map_err(|e| {
					HttpError::InvalidUrl(self.from_field, InvalidUrlError(e, s.to_owned()))
				})
			})?,
		};

		let url = url.ok_or_else(|| HttpError::MissingUrl(self.from_field))?;

		let new_page = sources::http::send_request(&self.client, &Request::Get, &url).await?;

		Ok(vec![TransformedEntry {
			raw_contents: TransformResult::New(new_page),
			msg: TransformedMessage {
				link: TransformResult::New(url.as_str().to_owned()),
				..Default::default()
			},
			..Default::default()
		}])
	}
}