#![allow(
clippy::module_name_repetitions,
clippy::struct_excessive_bools,
clippy::default_trait_access,
clippy::used_underscore_binding
)]
use std::{collections::HashSet, time::Duration};
use check_if_email_exists::{check_email, CheckEmailInput, Reachable};
use http::{
header::{HeaderMap, HeaderValue},
StatusCode,
};
use octocrab::Octocrab;
use regex::RegexSet;
use reqwest::{header, Url};
use secrecy::{ExposeSecret, SecretString};
use tokio::time::sleep;
use typed_builder::TypedBuilder;
use crate::{
filter::{Excludes, Filter, Includes},
quirks::Quirks,
remap::Remaps,
types::{mail, uri::github::GithubUri},
ErrorKind, Request, Response, Result, Status, Uri,
};
pub const DEFAULT_MAX_REDIRECTS: usize = 5;
pub const DEFAULT_MAX_RETRIES: u64 = 3;
pub const DEFAULT_RETRY_WAIT_TIME_SECS: usize = 1;
pub const DEFAULT_TIMEOUT_SECS: usize = 20;
pub const DEFAULT_USER_AGENT: &str = concat!("lychee/", env!("CARGO_PKG_VERSION"));
const CONNECT_TIMEOUT: u64 = 10;
const TCP_KEEPALIVE: u64 = 60;
#[derive(TypedBuilder, Debug, Clone)]
#[builder(field_defaults(default, setter(into)))]
#[builder(builder_method_doc = "
Create a builder for building `ClientBuilder`.
On the builder call, call methods with same name as its fields to set their values.
Finally, call `.build()` to create the instance of `ClientBuilder`.
")]
pub struct ClientBuilder {
github_token: Option<SecretString>,
remaps: Option<Remaps>,
includes: Option<RegexSet>,
excludes: Option<RegexSet>,
exclude_all_private: bool,
exclude_private_ips: bool,
exclude_link_local_ips: bool,
exclude_loopback_ips: bool,
exclude_mail: bool,
#[builder(default = DEFAULT_MAX_REDIRECTS)]
max_redirects: usize,
#[builder(default = DEFAULT_MAX_RETRIES)]
max_retries: u64,
#[builder(default_code = "String::from(DEFAULT_USER_AGENT)")]
user_agent: String,
allow_insecure: bool,
schemes: HashSet<String>,
custom_headers: HeaderMap,
#[builder(default = reqwest::Method::GET)]
method: reqwest::Method,
accepted: Option<HashSet<StatusCode>>,
timeout: Option<Duration>,
retry_wait_time: Option<Duration>,
require_https: bool,
}
impl Default for ClientBuilder {
#[must_use]
#[inline]
fn default() -> Self {
Self::builder().build()
}
}
impl ClientBuilder {
pub fn client(self) -> Result<Client> {
let Self {
github_token,
remaps,
includes,
excludes,
user_agent,
schemes,
custom_headers: mut headers,
method,
accepted,
..
} = self;
headers.insert(header::USER_AGENT, HeaderValue::from_str(&user_agent)?);
headers.insert(
header::TRANSFER_ENCODING,
HeaderValue::from_static("chunked"),
);
let builder = reqwest::ClientBuilder::new()
.gzip(true)
.default_headers(headers)
.danger_accept_invalid_certs(self.allow_insecure)
.connect_timeout(Duration::from_secs(CONNECT_TIMEOUT))
.tcp_keepalive(Duration::from_secs(TCP_KEEPALIVE))
.redirect(reqwest::redirect::Policy::limited(self.max_redirects));
let reqwest_client = (match self.timeout {
Some(t) => builder.timeout(t),
None => builder,
})
.build()
.map_err(ErrorKind::NetworkRequest)?;
let github_client = match github_token.as_ref().map(ExposeSecret::expose_secret) {
Some(token) if !token.is_empty() => Some(
Octocrab::builder()
.personal_token(token.clone())
.build()
.map_err(ErrorKind::BuildGithubClient)?,
),
_ => None,
};
let filter = Filter {
includes: includes.map(|regex| Includes { regex }),
excludes: excludes.map(|regex| Excludes { regex }),
schemes,
exclude_private_ips: self.exclude_all_private || self.exclude_private_ips,
exclude_link_local_ips: self.exclude_all_private || self.exclude_link_local_ips,
exclude_loopback_ips: self.exclude_all_private || self.exclude_loopback_ips,
exclude_mail: self.exclude_mail,
};
let retry_wait_time = self
.retry_wait_time
.unwrap_or_else(|| Duration::from_secs(DEFAULT_RETRY_WAIT_TIME_SECS as u64));
let quirks = Quirks::default();
Ok(Client {
reqwest_client,
github_client,
remaps,
filter,
max_retries: self.max_retries,
retry_wait_time,
method,
accepted,
require_https: self.require_https,
quirks,
})
}
}
#[derive(Debug, Clone)]
pub struct Client {
reqwest_client: reqwest::Client,
github_client: Option<Octocrab>,
remaps: Option<Remaps>,
filter: Filter,
max_retries: u64,
retry_wait_time: Duration,
method: reqwest::Method,
accepted: Option<HashSet<StatusCode>>,
require_https: bool,
quirks: Quirks,
}
impl Client {
pub async fn check<T, E>(&self, request: T) -> Result<Response>
where
Request: TryFrom<T, Error = E>,
ErrorKind: From<E>,
{
let Request { uri, source, .. } = request.try_into()?;
let uri = self.remap(uri)?;
let status = if self.filter.is_excluded(&uri) {
Status::Excluded
} else if uri.is_file() {
self.check_file(&uri)
} else if uri.is_mail() {
self.check_mail(&uri).await
} else {
match self.check_website(&uri).await {
Status::Ok(code) if self.require_https && uri.scheme() == "http" => {
let mut https_uri = uri.clone();
https_uri
.set_scheme("https")
.map_err(|_| ErrorKind::InvalidURI(uri.clone()))?;
if self.check_website(&https_uri).await.is_success() {
Status::Error(ErrorKind::InsecureURL(https_uri))
} else {
Status::Ok(code)
}
}
s => s,
}
};
Ok(Response::new(uri.clone(), status, source))
}
pub fn remap(&self, uri: Uri) -> Result<Uri> {
match self.remaps {
Some(ref remaps) => remaps.remap(uri),
None => Ok(uri),
}
}
#[must_use]
pub fn is_excluded(&self, uri: &Uri) -> bool {
self.filter.is_excluded(uri)
}
pub async fn check_website(&self, uri: &Uri) -> Status {
if invalid(&uri.url) {
if matches!(uri.scheme(), "http" | "https") {
return Status::Error(ErrorKind::InvalidURI(uri.clone()));
}
return Status::Unsupported(ErrorKind::InvalidURI(uri.clone()));
}
let mut retries: u64 = 0;
let mut wait = self.retry_wait_time;
let mut status = self.check_default(uri).await;
while retries < self.max_retries {
if status.is_success() {
return status;
}
sleep(wait).await;
retries += 1;
wait *= 2;
status = self.check_default(uri).await;
}
if let Ok(github_uri) = GithubUri::try_from(uri) {
let status = self.check_github(github_uri).await;
if status.is_success() {
return status;
}
}
status
}
async fn check_github(&self, uri: GithubUri) -> Status {
let client = match &self.github_client {
Some(client) => client,
None => return ErrorKind::MissingGitHubToken.into(),
};
let repo = match client.repos(&uri.owner, &uri.repo).get().await {
Ok(repo) => repo,
Err(e) => return ErrorKind::GithubRequest(e).into(),
};
if let Some(true) = repo.private {
return Status::Ok(StatusCode::OK);
} else if let Some(endpoint) = uri.endpoint {
return ErrorKind::InvalidGithubUrl(format!("{}/{}/{}", uri.owner, uri.repo, endpoint))
.into();
}
Status::Ok(StatusCode::OK)
}
async fn check_default(&self, uri: &Uri) -> Status {
let request = match self
.reqwest_client
.request(self.method.clone(), uri.as_str())
.build()
{
Ok(r) => r,
Err(e) => return e.into(),
};
let request = self.quirks.apply(request);
match self.reqwest_client.execute(request).await {
Ok(ref response) => Status::new(response, self.accepted.clone()),
Err(e) => e.into(),
}
}
pub fn check_file(&self, uri: &Uri) -> Status {
if let Ok(path) = uri.url.to_file_path() {
if path.exists() {
return Status::Ok(StatusCode::OK);
}
}
ErrorKind::InvalidFilePath(uri.clone()).into()
}
pub async fn check_mail(&self, uri: &Uri) -> Status {
let address = uri.url.path().to_string();
let input = CheckEmailInput::new(address);
let result = &(check_email(&input).await);
if let Reachable::Invalid = result.is_reachable {
ErrorKind::UnreachableEmailAddress(uri.clone(), mail::error_from_output(result)).into()
} else {
Status::Ok(StatusCode::OK)
}
}
}
fn invalid(url: &Url) -> bool {
url.as_str().parse::<http::Uri>().is_err()
}
pub async fn check<T, E>(request: T) -> Result<Response>
where
Request: TryFrom<T, Error = E>,
ErrorKind: From<E>,
{
let client = ClientBuilder::builder().build().client()?;
client.check(request).await
}
#[cfg(test)]
mod tests {
use std::{
fs::File,
time::{Duration, Instant},
};
use http::{header::HeaderMap, StatusCode};
use reqwest::header;
use tempfile::tempdir;
use super::ClientBuilder;
use crate::{mock_server, test_utils::get_mock_client_response, Uri};
#[tokio::test]
async fn test_nonexistent() {
let mock_server = mock_server!(StatusCode::NOT_FOUND);
let res = get_mock_client_response(mock_server.uri()).await;
assert!(res.status().is_failure());
}
#[tokio::test]
async fn test_nonexistent_with_path() {
let res = get_mock_client_response("http://127.0.0.1/invalid").await;
assert!(res.status().is_failure());
}
#[tokio::test]
async fn test_exponential_backoff() {
let mock_server = mock_server!(StatusCode::NOT_FOUND);
let start = Instant::now();
let res = get_mock_client_response(mock_server.uri()).await;
let end = start.elapsed();
assert!(res.status().is_failure());
assert!(end.as_secs() >= 7);
assert!(end.as_secs() <= 8);
}
#[tokio::test]
async fn test_github() {
let res = get_mock_client_response("https://github.com/lycheeverse/lychee").await;
assert!(res.status().is_success());
}
#[tokio::test]
async fn test_github_nonexistent_repo() {
let res = get_mock_client_response("https://github.com/lycheeverse/not-lychee").await;
assert!(res.status().is_failure());
}
#[tokio::test]
async fn test_github_nonexistent_file() {
let res = get_mock_client_response(
"https://github.com/lycheeverse/lychee/blob/master/NON_EXISTENT_FILE.md",
)
.await;
assert!(res.status().is_failure());
}
#[tokio::test]
async fn test_youtube() {
let res = get_mock_client_response("https://www.youtube.com/watch?v=NlKuICiT470&list=PLbWDhxwM_45mPVToqaIZNbZeIzFchsKKQ&index=7").await;
assert!(res.status().is_success());
let res = get_mock_client_response("https://www.youtube.com/watch?v=invalidNlKuICiT470&list=PLbWDhxwM_45mPVToqaIZNbZeIzFchsKKQ&index=7").await;
assert!(res.status().is_failure());
}
#[tokio::test]
async fn test_non_github() {
let mock_server = mock_server!(StatusCode::OK);
let res = get_mock_client_response(mock_server.uri()).await;
assert!(res.status().is_success());
}
#[tokio::test]
async fn test_invalid_ssl() {
let res = get_mock_client_response("https://expired.badssl.com/").await;
assert!(res.status().is_failure());
let res = ClientBuilder::builder()
.allow_insecure(true)
.build()
.client()
.unwrap()
.check("https://expired.badssl.com/")
.await
.unwrap();
assert!(res.status().is_success());
}
#[tokio::test]
async fn test_file() {
let dir = tempdir().unwrap();
let file = dir.path().join("temp");
File::create(file).unwrap();
let uri = format!("file://{}", dir.path().join("temp").to_str().unwrap());
let res = get_mock_client_response(uri).await;
assert!(res.status().is_success());
}
#[tokio::test]
async fn test_custom_headers() {
let mut custom = HeaderMap::new();
custom.insert(header::ACCEPT, "text/html".parse().unwrap());
let res = ClientBuilder::builder()
.custom_headers(custom)
.build()
.client()
.unwrap()
.check("https://crates.io/crates/lychee")
.await
.unwrap();
assert!(res.status().is_success());
}
#[tokio::test]
async fn test_exclude_mail() {
let client = ClientBuilder::builder()
.exclude_mail(false)
.exclude_all_private(true)
.build()
.client()
.unwrap();
assert!(!client.is_excluded(&Uri {
url: "mailto://mail@example.com".try_into().unwrap()
}));
let client = ClientBuilder::builder()
.exclude_mail(true)
.exclude_all_private(true)
.build()
.client()
.unwrap();
assert!(client.is_excluded(&Uri {
url: "mailto://mail@example.com".try_into().unwrap()
}));
}
#[tokio::test]
async fn test_require_https() {
let client = ClientBuilder::builder().build().client().unwrap();
let res = client.check("http://example.com").await.unwrap();
assert!(res.status().is_success());
let client = ClientBuilder::builder()
.require_https(true)
.build()
.client()
.unwrap();
let res = client.check("http://example.com").await.unwrap();
assert!(res.status().is_failure());
}
#[tokio::test]
async fn test_timeout() {
let mock_delay = Duration::from_millis(20);
let checker_timeout = Duration::from_millis(10);
assert!(mock_delay > checker_timeout);
let mock_server = mock_server!(StatusCode::OK, set_delay(mock_delay));
let client = ClientBuilder::builder()
.timeout(checker_timeout)
.build()
.client()
.unwrap();
let res = client.check(mock_server.uri()).await.unwrap();
assert!(res.status().is_timeout());
}
#[tokio::test]
async fn test_avoid_reqwest_panic() {
let client = ClientBuilder::builder().build().client().unwrap();
let res = client.check("http://\"").await.unwrap();
assert!(res.status().is_failure());
}
}