tag2upload_service_manager/

config.rs

use crate::prelude::*;

define_derive_deftly! {
    DefaultViaSerde:

    impl Default for $ttype {
        fn default() -> $ttype {
            serde_json::from_value(json!({})).expect("defaults all OK")
        }
    }
}

#[derive(Deserialize, derive_more::Debug)]
pub struct Config {
    pub t2u: T2u,

    #[serde(default)]
    pub intervals: Intervals,

    #[serde(default)]
    pub timeouts: Timeouts,

    #[serde(default)]
    pub limits: Limits,

    pub files: Files,

    pub vhosts: ui_vhost::Vhosts,

    #[serde(default)]
    pub log: Log,

    #[serde(default)]
    pub testing: Testing,

    #[serde(default)]
    pub retry: RetryPolicy,

    // NB, see the special transformations additions we make in `startup`;
    // just deserialising this and passing a `rocket::Config` to rocket
    // produces a non-working config.
    #[debug(skip)]
    pub rocket: rocket::Config,
}

/// Configuration information computed from `Config`
#[derive(Deserialize, Debug)]
pub struct ComputedConfig {
    pub unified_webhook_acl: Vec<dns::AllowedClient>,
    pub bsql_timeout: bsql::Timeout,
}

#[derive(Deserialize, Debug, Deftly)]
#[derive_deftly(DefaultViaSerde)]
pub struct Testing {
    /// Amount to adjust the wall clock by
    ///
    /// Positive values get added to the real clodk,
    /// meaning we'll think we're running in the future.
    ///
    /// Usually, this would be set to a negative value
    /// since test data is in the past.
    //
    // See also `test::GlobalSupplement.simulated_time_advance`,
    // which is mutable (and is added to this).
    #[serde(default)]
    pub time_offset: i64,

    /// Bodge the forge URL in the webhook request
    ///
    /// Forge URLs are expected to be
    /// `file://<fake_https_dir>/<hostname>/...`
    /// instead of `https://<hostname>/...`.
    ///
    /// API call are made by reading files under this directory
    /// rather than by making queries to `https://`.
    ///
    // See also `test::GlobalSupplement.url_map`, which can do
    // arbitrary prefix mapping on requests we make, but no
    // inbound URL mapping.
    pub fake_https_dir: Option<String>,

    /// Only allow source packages matching these glob patterns
    ///
    /// `None` means allow any.
    #[serde(default)]
    pub allowed_source_packages: Option<Vec<AllowedSourcePackage>>,
}

#[derive(Debug, Clone, Deref, Deftly)]
#[derive_deftly(DeserializeViaFromStr)]
#[deftly(deser(inner, expect = "glob pattern"))]
pub struct AllowedSourcePackage(glob::Pattern);

#[derive(Deserialize, Debug)]
pub struct T2u {
    pub distro: String,
    pub forges: Vec<Forge>,
}

#[derive(Deserialize, Debug, Deftly)]
#[derive_deftly(DefaultViaSerde)]
pub struct Intervals {
    #[serde(default = "days::<3>")]
    pub max_tag_age: HtDuration,

    #[serde(default = "secs::<1000>")]
    pub max_tag_age_skew: HtDuration,

    #[serde(default = "days::<32>")]
    pub expire: HtDuration,

    #[serde(default = "hours::<5>")]
    pub expire_every: HtDuration,

    #[serde(default = "days::<1>")]
    pub show_recent: HtDuration,
}

#[derive(Deserialize, Debug, Deftly)]
pub struct Files {
    pub db: String,

    pub o2m_socket: String,

    pub scratch_dir: Option<String>,

    pub archive_dir: String,

    /// If not set, uses compiled-in templates
    ///
    /// Useful during dev, for quick turnaround while editing templates.
    /// NB, this is not automatically reloaded.
    pub template_dir: Option<String>,

    /// Write the port to this file (in decimal) when we've finished startup
    ///
    /// The file is only opened when we're ready.
    /// If opening or writing fails, the t2usm immediately crashes.
    pub port_report_file: Option<String>,

    /// `.git` directory for us ourselves
    ///
    /// Used for `maint/build-repro describe`
    pub self_git_dir: Option<String>,
}

#[derive(Deserialize, Debug, Deftly)]
#[derive_deftly(DefaultViaSerde)]
pub struct Log {
    /// Log message level(s) to generate
    ///
    /// Default is:
    ///  * If `dir` is specified, the level needed by `schedule`
    ///  * Otherwise, `INFO`
    #[serde(with = "serde_log_level")]
    #[serde(default)]
    pub level: Option<logging::LevelFilter>,

    /// Extra tracing filter directives
    ///
    /// See
    /// <https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives>
    ///
    /// Appended to the default, which is
    /// `tag2upload_service_manager=LEVEL,info,"
    /// where LEVEL is the value `level`, above.
    #[serde(default)]
    pub tracing: String,

    /// Log directory
    ///
    /// If unspecified, use stdout.
    ///
    /// Files here will be named `t2usm.YYYY-MM-DD.log`
    /// and kept for `days` days.
    pub dir: Option<String>,

    /// How much keep and when to rotate, for each level
    ///
    /// Ignored if `dir` is not set.
    #[serde(default)]
    pub schedule: tracing_logrotate::Config,
}

#[derive(Deserialize, Debug, Deftly)]
#[derive_deftly(DefaultViaSerde)]
pub struct Timeouts {
    #[serde(default = "secs::<100>")]
    pub http_request: HtDuration,

    #[serde(default = "secs::<100>")]
    pub git_query: HtDuration,

    #[serde(default = "secs::<500>")]
    pub git_clone: HtDuration,

    #[serde(default = "secs::<10>")]
    pub unpause_poll: HtDuration,

    /// Should be > the oracle's worker restart timeout
    #[serde(default = "secs::<100>")]
    pub disconnected_worker_expire: HtDuration,

    /// Interval to check if the `o2m_socket` is removed, and if so exit
    ///
    /// This is a backstop cleanup approach for the dgit test suite.
    #[serde(default)]
    pub socket_stat_interval: Option<HtDuration>,

    /// sqlite overall database timeout
    #[serde(default = "secs::<10>")]
    pub db_timeout: HtDuration,

    /// sqlite retries
    ///
    /// Number of times we can get BUSY before giving up.
    /// This is the maximum number of deadlocks we'll tolerate.
    ///
    /// Each attempt will have a timeout of `db_timeout`/`db_retries`.
    #[serde(default = "u32_::<100>")]
    pub db_retries: u32,
}

#[derive(Deserialize, Debug, Deftly)]
#[derive_deftly(DefaultViaSerde)]
pub struct RetryPolicy {
    #[serde(default = "u32_::<15>")]
    pub min_retries: u32,
    #[serde(default = "u32_::<10>")]
    pub min_salient_retries: u32,
    #[serde(default = "secs::<100>")]
    pub timeout_initial: HtDuration,
    #[serde(default = "f32_per_mil::<12_000>")]
    pub timeout_increase: f32,
    #[serde(default = "hours::<12>")]
    pub timeout_mintotal: HtDuration,
}

#[derive(Deserialize, Debug, Deftly)]
#[derive_deftly(DefaultViaSerde)]
pub struct Limits {
    #[serde(default = "usize_::<16384>")]
    pub o2m_line: usize,
}

#[derive(Deserialize, Debug)]
pub struct Forge {
    pub host: Hostname,
    pub kind: String,
    pub allow: Vec<dns::AllowedClient>,
    #[serde(default = "u32_::<3>")]
    pub max_concurrent_fetch: u32,
}

const MAX_MAX_CONCURRENT_FETCH: u32 = 16;

//---------- impls (eg defaults) ----------

type HtD = HtDuration;

// serde default requires a path, but we can trick it with const generics
fn htd_from_secs(secs: u64) -> HtD { Duration::from_secs(secs).into() }
pub fn secs<const SECS: u64>() -> HtD { htd_from_secs(SECS        ) }
pub fn hours<const HRS: u64>() -> HtD { htd_from_secs( HRS * 3600 ) }
pub fn days<const DAYS: u64>() -> HtD { htd_from_secs(DAYS * 86400) }
// sadly Rust won't let us make this generic
pub fn u32_  <const U: u32  >() -> u32   { U }
pub fn usize_<const U: usize>() -> usize { U }
pub fn f32_per_mil<const M: u64>() -> f32 { M as f32 / 1000. }

impl Config {
    pub fn check(&self) -> Result<(), StartupError> {
        let mut errs = vec![];
        self.t2u.check_inner(&mut errs);
        self.intervals.check_inner(&mut errs);
        self.testing.check_inner(&mut errs);
        self.files.check_inner(&mut errs);

        if errs.is_empty() {
            return Ok(());
        }
        for e in errs {
            eprintln!("configuration error: {e:#}");
        }
        Err(StartupError::InvalidConfig)
    }
}

impl Files {
    fn check_inner(&self, errs: &mut Vec<AE>) {
        let archive_dir = &self.archive_dir;
        match (|| {
            let md = fs::metadata(archive_dir).context("stat")?;
            if !md.is_dir() {
                return Err(anyhow!("is not a directory"));
            }
            unix_access(&archive_dir, libc::W_OK | libc::X_OK)
                .context("check writeability")?;
            Ok(())
        })() {
            Err(e) => errs.push(
                e
                    .context(archive_dir.clone())
                    .context("config.files.archive_dir")
            ),
            Ok(()) => {},
        }
    }
}

impl T2u {
    fn check_inner(&self, errs: &mut Vec<AE>) {
        if self.forges.is_empty() {
            errs.push(anyhow!("no forges configured!"));
        }
        for (host, kind) in self.forges.iter()
            .map(|f| (&f.host, &f.kind))
            .duplicates()
        {
            errs.push(anyhow!("duplicate forge kind and host {kind} {host}"));
        }
        for forge in &self.forges {
            forge.check_inner(errs);
        }
    }
}

impl Forge {
    fn check_inner(&self, errs: &mut Vec<AE>) {
        if self.max_concurrent_fetch > MAX_MAX_CONCURRENT_FETCH {
            errs.push(anyhow!(
                "forge {} max_concurrent_fetch={} > hardcoded limit {}",
                self.host, self. max_concurrent_fetch,
                MAX_MAX_CONCURRENT_FETCH,
            ));
        }
    }
}
    
impl Testing {
    fn check_inner(&self, errs: &mut Vec<AE>) {
        if let Some(fake) = &self.fake_https_dir {
            if !fake.starts_with('/') {
                errs.push(anyhow!("t2u.fake_https_dir must be absolute"));
            }
        }
    }
}

impl Intervals {
    fn check_inner(&self, errs: &mut Vec<AE>) {
        let Intervals { max_tag_age, max_tag_age_skew, expire, .. } = *self;
        let min_expire = HtDuration::from(
            max_tag_age.checked_add(*max_tag_age_skew)
                .unwrap_or_else(|| {
                    errs.push(anyhow!(
 "max_tag_age and/or max_tag_age_slew far too large"
                    ));
                    Duration::ZERO
                })
        );
        if !(*expire > *min_expire) {
            errs.push(anyhow!(
                "expiry {expire} too short, must be > max_tag_age {max_tag_age} +  max_tag_age_skew {max_tag_age_skew}, > {min_expire}"
            ))
        }
    }
}

impl TryFrom<&Config> for ComputedConfig {
    type Error = StartupError;
    fn try_from(config: &Config) -> Result<ComputedConfig, StartupError> {
        let unified_webhook_acl = config.t2u.forges.iter()
            .map(|forge| &forge.allow)
            .flatten()
            .cloned()
            .collect_vec();
        let bsql_timeout = bsql::Timeout::new(
            *config.timeouts.db_timeout,
            config.timeouts.db_retries,
        );
        Ok(ComputedConfig {
            unified_webhook_acl,
            bsql_timeout,
        })
    }
}

#[test]
fn timeouts_defaults() {
    let _: Timeouts = Timeouts::default();
}

mod serde_log_level {
    use super::*;
    use logging::*;

    pub(super) fn deserialize<'de, D: Deserializer<'de>>(
        deser: D,
    ) -> Result<Option<LevelFilter>, D::Error> {
        let s: String = String::deserialize(deser)?.to_ascii_uppercase();
        let l: LevelFilter = s.parse()
            .map_err(|_| D::Error::invalid_value(
                serde::de::Unexpected::Str(&s),
                &"log level",
            ))?;
        Ok(Some(l))
    }
}