apt_swarm/
fetch.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
use crate::config;
use crate::config::Repository;
use crate::db::DatabaseClient;
use crate::errors::*;
use crate::keyring::Keyring;
use crate::signed::Signed;
use sequoia_openpgp::Fingerprint;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use tokio::task::JoinSet;

pub const DEFAULT_CONCURRENCY: usize = 4;
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
pub const READ_TIMEOUT: Duration = Duration::from_secs(60);

async fn fetch_repository_updates(
    client: &reqwest::Client,
    keyring: &Option<Keyring>,
    repository: &config::Repository,
) -> Result<Vec<(Option<Fingerprint>, Signed)>> {
    let mut out = Vec::new();

    for source in &repository.urls {
        let Ok(signed) = source
            .fetch(client)
            .await
            .inspect_err(|err| error!("Error fetching latest release: {err:#}"))
        else {
            continue;
        };

        for item in signed.canonicalize(keyring.as_ref())? {
            out.push(item);
        }
    }

    Ok(out)
}

pub async fn fetch_updates<D: DatabaseClient>(
    db: &mut D,
    keyring: Arc<Option<Keyring>>,
    concurrency: Option<usize>,
    repositories: Vec<Repository>,
    proxy: Option<SocketAddr>,
) -> Result<()> {
    let concurrency = concurrency.unwrap_or(DEFAULT_CONCURRENCY);
    let mut queue = repositories.into_iter();
    let mut pool = JoinSet::new();
    let mut client = reqwest::Client::builder()
        .connect_timeout(CONNECT_TIMEOUT)
        .read_timeout(READ_TIMEOUT);
    if let Some(proxy) = proxy {
        let proxy = format!("socks5h://{proxy:?}");
        let proxy = reqwest::Proxy::all(&proxy)
            .with_context(|| anyhow!("Failed to parse as proxy: {proxy:?}"))?;
        client = client.proxy(proxy);
    }
    let client = client.build().context("Failed to setup http client")?;

    loop {
        while pool.len() < concurrency {
            if let Some(repository) = queue.next() {
                let client = client.clone();
                let keyring = keyring.clone();
                pool.spawn(async move {
                    fetch_repository_updates(&client, &keyring, &repository).await
                });
            } else {
                // no more tasks to schedule
                break;
            }
        }
        if let Some(join) = pool.join_next().await {
            match join.context("Failed to join task")? {
                Ok(list) => {
                    for (fp, variant) in list {
                        let fp = fp.context(
                            "Signature can't be imported because the signature is unverified",
                        )?;
                        db.add_release(&fp, &variant).await?;
                    }
                }
                Err(err) => error!("Error fetching latest release: {err:#}"),
            }
        } else {
            // no more tasks in pool
            break;
        }
    }

    Ok(())
}