muffy/
config.rs

1mod error;
2mod serde;
3mod toml;
4
5pub use self::{
6    error::ConfigError,
7    serde::{SerializableConfig, compile_config},
8    toml::read_config,
9};
10use alloc::sync::Arc;
11use core::{cmp::Reverse, ops::Deref, time::Duration};
12use http::{HeaderMap, StatusCode};
13use regex::Regex;
14use rlimit::{Resource, getrlimit};
15use std::collections::{HashMap, HashSet};
16use url::Url;
17
18/// Default accepted URL schemes.
19pub const DEFAULT_ACCEPTED_SCHEMES: &[&str] = &["http", "https"];
20/// Default accepted HTTP status codes.
21pub const DEFAULT_ACCEPTED_STATUS_CODES: &[StatusCode] = &[StatusCode::OK];
22/// A default number of maximum redirects.
23pub const DEFAULT_MAX_REDIRECTS: usize = 16;
24/// A default HTTP timeout.
25pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
26
27const DEFAULT_MINIMUM_CONCURRENCY: usize = 256;
28
29/// Returns a default concurrency.
30pub fn default_concurrency() -> usize {
31    getrlimit(Resource::NOFILE)
32        .map(|(count, _)| (count / 2) as _)
33        .unwrap_or(DEFAULT_MINIMUM_CONCURRENCY)
34}
35
36/// A validation configuration.
37#[derive(Clone, Debug)]
38pub struct Config {
39    roots: Vec<String>,
40    ignored_links: Vec<Regex>,
41    default: Arc<SiteConfig>,
42    sites: HashMap<String, Vec<(String, Arc<SiteConfig>)>>,
43    concurrency: ConcurrencyConfig,
44    persistent_cache: bool,
45    rate_limit: RateLimitConfig,
46}
47
48impl Config {
49    /// Creates a configuration.
50    pub fn new(
51        roots: Vec<String>,
52        default: Arc<SiteConfig>,
53        sites: HashMap<String, HashMap<String, Arc<SiteConfig>>>,
54    ) -> Self {
55        Self {
56            roots,
57            ignored_links: Default::default(),
58            default,
59            sites: sites
60                .into_iter()
61                .map(|(host, value)| {
62                    let mut paths = value.into_iter().collect::<Vec<_>>();
63                    paths.sort_by_key(|(path, _)| Reverse(path.clone()));
64                    (host, paths)
65                })
66                .collect(),
67            concurrency: Default::default(),
68            persistent_cache: false,
69            rate_limit: Default::default(),
70        }
71    }
72
73    /// Returns root URLs.
74    pub fn roots(&self) -> impl Iterator<Item = &str> {
75        self.roots.iter().map(Deref::deref)
76    }
77
78    /// Returns ignored link patterns.
79    pub fn ignored_links(&self) -> impl Iterator<Item = &Regex> {
80        self.ignored_links.iter()
81    }
82
83    /// Returns sites.
84    pub const fn sites(&self) -> &HashMap<String, Vec<(String, Arc<SiteConfig>)>> {
85        &self.sites
86    }
87
88    /// Returns a site.
89    pub fn site(&self, url: &Url) -> &SiteConfig {
90        self.get_site(url).unwrap_or(&self.default)
91    }
92
93    /// Returns concurrency.
94    pub const fn concurrency(&self) -> &ConcurrencyConfig {
95        &self.concurrency
96    }
97
98    /// Returns whether a cache is persistent.
99    pub const fn persistent_cache(&self) -> bool {
100        self.persistent_cache
101    }
102
103    /// Returns a rate limit.
104    pub const fn rate_limit(&self) -> &RateLimitConfig {
105        &self.rate_limit
106    }
107
108    /// Sets concurrency.
109    pub fn set_concurrency(mut self, concurrency: ConcurrencyConfig) -> Self {
110        self.concurrency = concurrency;
111        self
112    }
113
114    /// Sets ignored link patterns.
115    pub fn set_ignored_links(mut self, links: Vec<Regex>) -> Self {
116        self.ignored_links = links;
117        self
118    }
119
120    /// Sets persistent cache.
121    pub const fn set_persistent_cache(mut self, persistent_cache: bool) -> Self {
122        self.persistent_cache = persistent_cache;
123        self
124    }
125
126    /// Sets a rate limit.
127    pub fn set_rate_limit(mut self, rate_limit: RateLimitConfig) -> Self {
128        self.rate_limit = rate_limit;
129        self
130    }
131
132    fn get_site(&self, url: &Url) -> Option<&SiteConfig> {
133        self.sites()
134            .get(url.host_str()?)?
135            .iter()
136            .find_map(|(path, config)| url.path().starts_with(path).then_some(config.as_ref()))
137    }
138}
139
140/// A site configuration.
141#[derive(Clone, Debug, Default, PartialEq)]
142pub struct SiteConfig {
143    id: Option<Arc<str>>,
144    cache: CacheConfig,
145    fragments_ignored: bool,
146    headers: HeaderMap,
147    max_redirects: usize,
148    recursive: bool,
149    retry: Arc<RetryConfig>,
150    scheme: SchemeConfig,
151    status: StatusConfig,
152    timeout: Option<Duration>,
153    validation: ValidationConfig,
154}
155
156impl SiteConfig {
157    /// Creates a site configuration.
158    pub fn new() -> Self {
159        Self::default()
160    }
161
162    /// Returns an ID.
163    pub const fn id(&self) -> Option<&Arc<str>> {
164        self.id.as_ref()
165    }
166
167    /// Returns a cache configuration.
168    pub const fn cache(&self) -> &CacheConfig {
169        &self.cache
170    }
171
172    /// Returns whether URL fragments should be ignored.
173    pub const fn fragments_ignored(&self) -> bool {
174        self.fragments_ignored
175    }
176
177    /// Returns headers attached to HTTP requests.
178    pub const fn headers(&self) -> &HeaderMap {
179        &self.headers
180    }
181
182    /// Returns a retry configuration.
183    pub const fn retry(&self) -> &Arc<RetryConfig> {
184        &self.retry
185    }
186
187    /// Returns a status code configuration.
188    pub const fn status(&self) -> &StatusConfig {
189        &self.status
190    }
191
192    /// Returns a scheme configuration.
193    pub const fn scheme(&self) -> &SchemeConfig {
194        &self.scheme
195    }
196
197    /// Returns a maximum number of redirects.
198    pub const fn max_redirects(&self) -> usize {
199        self.max_redirects
200    }
201
202    /// Returns a timeout.
203    pub const fn timeout(&self) -> Option<Duration> {
204        self.timeout
205    }
206
207    /// Returns whether we should validate the website recursively.
208    pub const fn recursive(&self) -> bool {
209        self.recursive
210    }
211
212    /// Returns a validation configuration.
213    pub const fn validation(&self) -> &ValidationConfig {
214        &self.validation
215    }
216
217    /// Sets an ID.
218    pub fn set_id(mut self, id: Option<Arc<str>>) -> Self {
219        self.id = id;
220        self
221    }
222
223    /// Sets a cache configuration.
224    pub const fn set_cache(mut self, cache: CacheConfig) -> Self {
225        self.cache = cache;
226        self
227    }
228
229    /// Sets whether URL fragments are ignored.
230    pub const fn set_fragments_ignored(mut self, ignored: bool) -> Self {
231        self.fragments_ignored = ignored;
232        self
233    }
234
235    /// Sets request headers.
236    pub fn set_headers(mut self, headers: HeaderMap) -> Self {
237        self.headers = headers;
238        self
239    }
240
241    /// Sets a retry configuration.
242    pub fn set_retry(mut self, retry: Arc<RetryConfig>) -> Self {
243        self.retry = retry;
244        self
245    }
246
247    /// Sets a status code configuration.
248    pub fn set_status(mut self, status: StatusConfig) -> Self {
249        self.status = status;
250        self
251    }
252
253    /// Sets a scheme configuration.
254    pub fn set_scheme(mut self, scheme: SchemeConfig) -> Self {
255        self.scheme = scheme;
256        self
257    }
258
259    /// Sets a maximum number of redirects.
260    pub const fn set_max_redirects(mut self, count: usize) -> Self {
261        self.max_redirects = count;
262        self
263    }
264
265    /// Sets a timeout.
266    pub const fn set_timeout(mut self, duration: Option<Duration>) -> Self {
267        self.timeout = duration;
268        self
269    }
270
271    /// Sets whether we should validate the website recursively
272    pub const fn set_recursive(mut self, recursive: bool) -> Self {
273        self.recursive = recursive;
274        self
275    }
276
277    /// Sets a validation configuration.
278    pub fn set_validation(mut self, validation: ValidationConfig) -> Self {
279        self.validation = validation;
280        self
281    }
282}
283
284/// A status code configuration.
285#[derive(Clone, Debug, Eq, PartialEq)]
286pub struct StatusConfig {
287    accepted: HashSet<StatusCode>,
288}
289
290impl StatusConfig {
291    /// Creates a status code configuration.
292    pub const fn new(accepted: HashSet<StatusCode>) -> Self {
293        Self { accepted }
294    }
295
296    /// Returns whether a status code is accepted.
297    pub fn accepted(&self, status: StatusCode) -> bool {
298        self.accepted.contains(&status)
299    }
300}
301
302impl Default for StatusConfig {
303    fn default() -> Self {
304        Self {
305            accepted: DEFAULT_ACCEPTED_STATUS_CODES.iter().copied().collect(),
306        }
307    }
308}
309
310/// A scheme configuration.
311#[derive(Clone, Debug, Eq, PartialEq)]
312pub struct SchemeConfig {
313    accepted: HashSet<String>,
314}
315
316impl SchemeConfig {
317    /// Creates a scheme configuration.
318    pub const fn new(accepted: HashSet<String>) -> Self {
319        Self { accepted }
320    }
321
322    /// Returns whether a scheme is accepted.
323    pub fn accepted(&self, scheme: &str) -> bool {
324        self.accepted.contains(scheme)
325    }
326}
327
328impl Default for SchemeConfig {
329    fn default() -> Self {
330        Self {
331            accepted: DEFAULT_ACCEPTED_SCHEMES
332                .iter()
333                .copied()
334                .map(ToOwned::to_owned)
335                .collect(),
336        }
337    }
338}
339
340/// A validation configuration.
341#[derive(Clone, Debug, Eq, PartialEq, Default)]
342pub struct ValidationConfig {
343    html: Option<MarkupConfig>,
344    svg: Option<MarkupConfig>,
345    css: bool,
346}
347
348impl ValidationConfig {
349    /// Returns an HTML validation configuration.
350    pub const fn html(&self) -> Option<&MarkupConfig> {
351        self.html.as_ref()
352    }
353
354    /// Returns an SVG validation configuration.
355    pub const fn svg(&self) -> Option<&MarkupConfig> {
356        self.svg.as_ref()
357    }
358
359    /// Returns whether CSS validation is enabled.
360    pub const fn css(&self) -> bool {
361        self.css
362    }
363
364    /// Sets an HTML validation configuration.
365    pub fn set_html(mut self, config: Option<MarkupConfig>) -> Self {
366        self.html = config;
367        self
368    }
369
370    /// Sets an SVG validation configuration.
371    pub fn set_svg(mut self, config: Option<MarkupConfig>) -> Self {
372        self.svg = config;
373        self
374    }
375
376    /// Sets whether CSS validation is enabled.
377    pub const fn set_css(mut self, enabled: bool) -> Self {
378        self.css = enabled;
379        self
380    }
381}
382
383/// A markup validation configuration.
384#[derive(Clone, Debug, Default)]
385pub struct MarkupConfig {
386    ignored_attributes: Vec<Regex>,
387    ignored_elements: Vec<Regex>,
388}
389
390impl MarkupConfig {
391    /// Creates a markup validation configuration.
392    pub const fn new(ignored_attributes: Vec<Regex>, ignored_elements: Vec<Regex>) -> Self {
393        Self {
394            ignored_attributes,
395            ignored_elements,
396        }
397    }
398
399    /// Returns ignored attributes.
400    pub fn ignored_attributes(&self) -> &[Regex] {
401        &self.ignored_attributes
402    }
403
404    /// Returns ignored elements.
405    pub fn ignored_elements(&self) -> &[Regex] {
406        &self.ignored_elements
407    }
408}
409
410impl PartialEq for MarkupConfig {
411    fn eq(&self, other: &Self) -> bool {
412        self.ignored_attributes.len() == other.ignored_attributes.len()
413            && self.ignored_elements.len() == other.ignored_elements.len()
414            && self
415                .ignored_attributes
416                .iter()
417                .zip(&other.ignored_attributes)
418                .chain(self.ignored_elements.iter().zip(&other.ignored_elements))
419                .all(|(one, other)| one.as_str() == other.as_str())
420    }
421}
422
423impl Eq for MarkupConfig {}
424
425/// A cache configuration.
426#[derive(Clone, Debug, Default, Eq, PartialEq)]
427pub struct CacheConfig {
428    max_age: Duration,
429    stale_while_revalidate: Duration,
430}
431
432impl CacheConfig {
433    /// Creates a cache configuration.
434    pub fn new() -> Self {
435        Self::default()
436    }
437
438    /// Returns a maximum age.
439    pub const fn max_age(&self) -> Duration {
440        self.max_age
441    }
442
443    /// Returns a stale-while-revalidate period.
444    pub const fn stale_while_revalidate(&self) -> Duration {
445        self.stale_while_revalidate
446    }
447
448    /// Sets a maximum age.
449    pub const fn set_max_age(mut self, age: Duration) -> Self {
450        self.max_age = age;
451        self
452    }
453
454    /// Sets a stale-while-revalidate period.
455    pub const fn set_stale_while_revalidate(mut self, period: Duration) -> Self {
456        self.stale_while_revalidate = period;
457        self
458    }
459}
460
461/// A retry configuration.
462#[derive(Clone, Debug, Default, PartialEq)]
463pub struct RetryConfig {
464    count: usize,
465    factor: f64,
466    interval: RetryDurationConfig,
467    statuses: HashSet<StatusCode>,
468}
469
470impl RetryConfig {
471    /// Creates a configuration.
472    pub fn new() -> Self {
473        Self {
474            count: 0,
475            factor: 1.0,
476            interval: Default::default(),
477            statuses: Default::default(),
478        }
479    }
480
481    /// Returns a count.
482    pub const fn count(&self) -> usize {
483        self.count
484    }
485
486    /// Returns a factor.
487    pub const fn factor(&self) -> f64 {
488        self.factor
489    }
490
491    /// Returns a duration configuration.
492    pub const fn interval(&self) -> &RetryDurationConfig {
493        &self.interval
494    }
495
496    /// Returns a set of status codes.
497    pub const fn statuses(&self) -> &HashSet<StatusCode> {
498        &self.statuses
499    }
500
501    /// Sets a count.
502    pub const fn set_count(mut self, count: usize) -> Self {
503        self.count = count;
504        self
505    }
506
507    /// Sets a factor.
508    pub const fn set_factor(mut self, factor: f64) -> Self {
509        self.factor = factor;
510        self
511    }
512
513    /// Sets a duration configuration.
514    pub const fn set_interval(mut self, duration: RetryDurationConfig) -> Self {
515        self.interval = duration;
516        self
517    }
518
519    /// Sets a set of status codes.
520    pub fn set_statuses(mut self, statuses: HashSet<StatusCode>) -> Self {
521        self.statuses = statuses;
522        self
523    }
524}
525
526/// A retry duration configuration.
527#[derive(Clone, Debug, Default, Eq, PartialEq)]
528pub struct RetryDurationConfig {
529    initial: Duration,
530    cap: Option<Duration>,
531}
532
533impl RetryDurationConfig {
534    /// Creates a configuration.
535    pub fn new() -> Self {
536        Self::default()
537    }
538
539    /// Returns an initial duration.
540    pub const fn initial(&self) -> Duration {
541        self.initial
542    }
543
544    /// Returns a cap duration.
545    pub const fn cap(&self) -> Option<Duration> {
546        self.cap
547    }
548
549    /// Sets an initial duration.
550    pub const fn set_initial(mut self, duration: Duration) -> Self {
551        self.initial = duration;
552        self
553    }
554
555    /// Sets a cap duration.
556    pub const fn set_cap(mut self, duration: Option<Duration>) -> Self {
557        self.cap = duration;
558        self
559    }
560}
561
562/// A concurrency configuration.
563#[derive(Clone, Debug, Default, Eq, PartialEq)]
564pub struct ConcurrencyConfig {
565    global: Option<usize>,
566    sites: HashMap<String, usize>,
567}
568
569impl ConcurrencyConfig {
570    /// Creates a configuration.
571    pub fn new() -> Self {
572        Self::default()
573    }
574
575    /// Returns a global concurrency.
576    pub const fn global(&self) -> Option<usize> {
577        self.global
578    }
579
580    /// Returns concurrency per site.
581    pub const fn sites(&self) -> &HashMap<String, usize> {
582        &self.sites
583    }
584
585    /// Sets a global concurrency.
586    pub const fn set_global(mut self, concurrency: Option<usize>) -> Self {
587        self.global = concurrency;
588        self
589    }
590
591    /// Sets concurrency per site.
592    pub fn set_sites(mut self, sites: HashMap<String, usize>) -> Self {
593        self.sites = sites;
594        self
595    }
596}
597
598/// A rate limit configuration.
599#[derive(Clone, Debug, Default, Eq, PartialEq)]
600pub struct RateLimitConfig {
601    global: Option<SiteRateLimitConfig>,
602    sites: HashMap<String, SiteRateLimitConfig>,
603}
604
605impl RateLimitConfig {
606    /// Creates a configuration.
607    pub fn new() -> Self {
608        Self::default()
609    }
610
611    /// Returns a global rate limit.
612    pub const fn global(&self) -> Option<&SiteRateLimitConfig> {
613        self.global.as_ref()
614    }
615
616    /// Returns rate limits per site.
617    pub const fn sites(&self) -> &HashMap<String, SiteRateLimitConfig> {
618        &self.sites
619    }
620
621    /// Sets a global rate limit.
622    pub const fn set_global(mut self, rate_limit: Option<SiteRateLimitConfig>) -> Self {
623        self.global = rate_limit;
624        self
625    }
626
627    /// Sets rate limits per site.
628    pub fn set_sites(mut self, sites: HashMap<String, SiteRateLimitConfig>) -> Self {
629        self.sites = sites;
630        self
631    }
632}
633
634/// A site rate limit configuration.
635#[derive(Clone, Debug, Default, Eq, PartialEq)]
636pub struct SiteRateLimitConfig {
637    supply: u64,
638    window: Duration,
639}
640
641impl SiteRateLimitConfig {
642    /// Creates a configuration.
643    pub const fn new(supply: u64, window: Duration) -> Self {
644        Self { supply, window }
645    }
646
647    /// Returns a supply.
648    pub const fn supply(&self) -> u64 {
649        self.supply
650    }
651
652    /// Returns a window.
653    pub const fn window(&self) -> Duration {
654        self.window
655    }
656
657    /// Sets a supply.
658    pub const fn set_supply(mut self, supply: u64) -> Self {
659        self.supply = supply;
660        self
661    }
662
663    /// Sets a window.
664    pub const fn set_window(mut self, window: Duration) -> Self {
665        self.window = window;
666        self
667    }
668}
669
670#[cfg(test)]
671mod tests {
672    use super::*;
673
674    #[test]
675    fn site_config_path_order() {
676        let config = Config::new(
677            vec![],
678            Default::default(),
679            [(
680                "example.com".to_string(),
681                [
682                    (
683                        "/foo".to_string(),
684                        SiteConfig::default()
685                            .set_id(Some("foo".into()))
686                            .set_recursive(true)
687                            .into(),
688                    ),
689                    (
690                        "/bar".to_string(),
691                        SiteConfig::default()
692                            .set_id(Some("bar".into()))
693                            .set_recursive(true)
694                            .into(),
695                    ),
696                    (
697                        "/".to_string(),
698                        SiteConfig::default()
699                            .set_id(Some("top".into()))
700                            .set_recursive(false)
701                            .into(),
702                    ),
703                    (
704                        "/baz".to_string(),
705                        SiteConfig::default()
706                            .set_id(Some("baz".into()))
707                            .set_recursive(true)
708                            .into(),
709                    ),
710                    (
711                        "/qux".to_string(),
712                        SiteConfig::default()
713                            .set_id(Some("qux".into()))
714                            .set_recursive(true)
715                            .into(),
716                    ),
717                ]
718                .into_iter()
719                .collect(),
720            )]
721            .into(),
722        );
723
724        assert!(
725            config
726                .site(&Url::parse("http://example.com/foo").unwrap())
727                .recursive()
728        );
729        assert!(
730            config
731                .site(&Url::parse("http://example.com/bar").unwrap())
732                .recursive()
733        );
734        assert!(
735            config
736                .site(&Url::parse("http://example.com/baz").unwrap())
737                .recursive()
738        );
739        assert!(
740            config
741                .site(&Url::parse("http://example.com/qux").unwrap())
742                .recursive()
743        );
744        assert!(
745            !config
746                .site(&Url::parse("http://example.com/other").unwrap())
747                .recursive()
748        );
749    }
750
751    #[test]
752    fn default_validation_config() {
753        let config = ValidationConfig::default();
754
755        assert!(config.html().is_none());
756        assert!(config.svg().is_none());
757        assert!(!config.css());
758    }
759
760    #[test]
761    fn set_validation_config_enabled() {
762        let config = ValidationConfig::default()
763            .set_html(Some(MarkupConfig::default()))
764            .set_svg(Some(MarkupConfig::default()))
765            .set_css(true);
766
767        assert!(config.html().is_some());
768        assert!(config.svg().is_some());
769        assert!(config.css());
770    }
771
772    #[test]
773    fn validate_site_config() {
774        let config = SiteConfig::default();
775
776        assert!(config.validation().html().is_none());
777        assert!(
778            config
779                .set_validation(ValidationConfig::default().set_html(Some(MarkupConfig::default())))
780                .validation()
781                .html()
782                .is_some()
783        );
784    }
785
786    #[test]
787    fn retry_config_statuses() {
788        let config = RetryConfig::new().set_statuses(HashSet::from([StatusCode::REQUEST_TIMEOUT]));
789
790        assert_eq!(
791            config.statuses(),
792            &HashSet::from([StatusCode::REQUEST_TIMEOUT])
793        );
794    }
795}
muffy/config.rs

muffy/
config.rs
