Skip to main content

passalong_https/
store.rs

1//! [`HttpStore`]: a passalong-server workspace as a [`Store`].
2//!
3//! Items have the bytes `FsStore` gives them, through `store::format`: the
4//! server keeps `meta` and the content as they are sent. In a sealed
5//! workspace both are sealed here, and everything read is opened and checked
6//! here; the server authenticates nothing about content.
7//!
8//! Every write names the key the client believes the workspace is under
9//! (`expectedKeyId`), so the server refuses it atomically when that changed.
10
11use std::io;
12use std::sync::Arc;
13use std::time::Duration;
14
15use async_trait::async_trait;
16use bytes::Bytes;
17use passalong_core::clock::Clock;
18use passalong_core::crypto::{self, KeyId, Sealer};
19use passalong_core::encryption::EncryptionError;
20use passalong_core::fs::BoxRead;
21use passalong_core::model::{ContentDigest, ContentKey, ItemId, ItemMeta, NewItem};
22use passalong_core::store::format::{self, CopyError, MetaError};
23use passalong_core::store::{PutOutcome, Store, StoreError, WriteProbe};
24use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, RANGE};
25use reqwest::{StatusCode, Url};
26use serde_json::value::RawValue;
27use tokio::sync::OnceCell;
28
29use crate::api::{self, BeginUpload, Cleaned, Resolved, UploadTicket};
30use crate::client::{Client, json};
31use crate::error::{Code, HttpsError};
32
33/// How many times a download that broke off is resumed.
34pub const MAX_RESUMES: u32 = 3;
35
36/// Which of a workspace's generations a store reads.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38#[non_exhaustive]
39pub enum Partition {
40    /// The workspace's items.
41    Current,
42    /// The items a fresh start set aside, unencrypted.
43    Plain,
44    /// An open rewrite's next generation; its holder alone reads it.
45    Staged,
46}
47
48impl Partition {
49    fn name(self) -> &'static str {
50        match self {
51            Self::Current => "current",
52            Self::Plain => "plain",
53            Self::Staged => "staged",
54        }
55    }
56}
57
58/// A passalong-server workspace.
59pub struct HttpStore {
60    client: Arc<Client>,
61    sealer: Option<Sealer>,
62    clock: Arc<dyn Clock>,
63    partition: Partition,
64    /// The server's `maxItemBytes`, read before the first upload.
65    limit: OnceCell<Option<u64>>,
66    /// The workspace's key id, which the plain partition's writes name.
67    workspace_key: OnceCell<Option<String>>,
68}
69
70impl std::fmt::Debug for HttpStore {
71    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        f.debug_struct("HttpStore")
73            .field("client", &self.client)
74            .field("sealed", &self.sealer.is_some())
75            .field("partition", &self.partition)
76            .finish_non_exhaustive()
77    }
78}
79
80/// The store error for a failed request: the key and rewrite codes as the
81/// encryption errors every backend gives, anything else in the server's
82/// words.
83pub fn store_error(err: HttpsError) -> StoreError {
84    match err.code() {
85        Some(Code::KeyIdMismatch) => EncryptionError::KeyChanged.into(),
86        Some(Code::RewriteInProgress) => EncryptionError::Rewriting { started: None }.into(),
87        Some(Code::Unauthenticated | Code::KeyExpired | Code::KeyRevoked) => StoreError::Denied(
88            format!("{err}; ask the server's operator for a new API key"),
89        ),
90        _ => StoreError::Backend(err.to_string()),
91    }
92}
93
94fn is_not_found(err: &HttpsError) -> bool {
95    err.code() == Some(Code::NotFound)
96}
97
98fn corrupt(id: &str, reason: impl std::fmt::Display) -> StoreError {
99    StoreError::Corrupt {
100        id: id.to_owned(),
101        reason: reason.to_string(),
102    }
103}
104
105impl HttpStore {
106    /// The workspace behind `client`: sealed with `sealer`, or plaintext.
107    pub fn new(client: Arc<Client>, sealer: Option<Sealer>, clock: Arc<dyn Clock>) -> Self {
108        Self::in_partition(client, sealer, clock, Partition::Current)
109    }
110
111    /// The workspace's `partition`.
112    pub fn in_partition(
113        client: Arc<Client>,
114        sealer: Option<Sealer>,
115        clock: Arc<dyn Clock>,
116        partition: Partition,
117    ) -> Self {
118        Self {
119            client,
120            sealer,
121            clock,
122            partition,
123            limit: OnceCell::new(),
124            workspace_key: OnceCell::new(),
125        }
126    }
127
128    /// The client this store sends with.
129    pub fn client(&self) -> &Arc<Client> {
130        &self.client
131    }
132
133    fn url(&self, path: &str, query: &[(&str, &str)]) -> Result<Url, StoreError> {
134        let mut url = Url::parse(&self.client.url(path))
135            .map_err(|err| StoreError::Config(format!("server.https.url: {err}")))?;
136        // Only the item routes have partitions; uploads name the rewrite in
137        // their body.
138        let item_route = path == "/items"
139            || path == "/item-ids"
140            || (path.starts_with("/items/") && path != "/items/resolve");
141        let partition =
142            (self.partition != Partition::Current && item_route).then(|| self.partition.name());
143        if partition.is_some() || !query.is_empty() {
144            let mut pairs = url.query_pairs_mut();
145            if let Some(partition) = partition {
146                pairs.append_pair("partition", partition);
147            }
148            for (name, value) in query {
149                pairs.append_pair(name, value);
150            }
151        }
152        Ok(url)
153    }
154
155    /// The key id every write names: the store's key, or for the plain
156    /// partition, which holds plaintext items in a sealed workspace, the
157    /// workspace's key.
158    async fn expected(&self) -> Result<Option<String>, StoreError> {
159        if self.partition != Partition::Plain {
160            return Ok(self
161                .sealer
162                .as_ref()
163                .map(|sealer| sealer.key_id().to_string()));
164        }
165        self.workspace_key
166            .get_or_try_init(|| async {
167                let workspace = self.client.workspace().await.map_err(store_error)?;
168                Ok::<_, StoreError>(workspace.encryption.key_id)
169            })
170            .await
171            .cloned()
172    }
173
174    /// Warns, as the filesystem store does, while items a fresh start set
175    /// aside remain unencrypted.
176    async fn remind_plain_left(&self) {
177        if self.sealer.is_none() || self.partition != Partition::Current {
178            return;
179        }
180        let Ok(mut url) = Url::parse(&self.client.url("/item-ids")) else {
181            return;
182        };
183        url.query_pairs_mut().append_pair("partition", "plain");
184        let Ok(response) = self
185            .client
186            .send("listItemIds", |http| http.get(url.clone()))
187            .await
188        else {
189            return;
190        };
191        let n = json::<Vec<String>>(response)
192            .await
193            .map_or(0, |ids| ids.len());
194        if n > 0 {
195            let (items, them) = if n == 1 {
196                ("item remains", "it")
197            } else {
198                ("items remain", "them")
199            };
200            tracing::warn!(
201                "{n} unencrypted {items} from before encryption; remove {them} with `passalong prune --plain`"
202            );
203        }
204    }
205
206    /// An envelope's metadata and, when sealed, its content salt.
207    fn decode(
208        &self,
209        item: &api::Item,
210    ) -> Result<(ItemMeta, Option<format::ContentSalt>), StoreError> {
211        let id = ItemId::parse(&item.id).map_err(|err| corrupt(&item.id, err))?;
212        format::decode_meta(&id, item.meta.get().as_bytes(), self.sealer.as_ref()).map_err(|err| {
213            match err {
214                // A server's item is complete when it is listed, so one
215                // that does not open is damaged, never still arriving.
216                MetaError::Corrupt(reason) | MetaError::Unopened(reason) => {
217                    corrupt(&item.id, reason)
218                }
219            }
220        })
221    }
222
223    async fn item(&self, id: &ItemId) -> Result<Option<api::Item>, StoreError> {
224        let url = self.url(&format!("/items/{id}"), &[])?;
225        match self
226            .client
227            .send("getItem", |http| http.get(url.clone()))
228            .await
229        {
230            Ok(response) => Ok(Some(json(response).await.map_err(store_error)?)),
231            Err(err) if is_not_found(&err) => Ok(None),
232            Err(err) => Err(store_error(err)),
233        }
234    }
235
236    async fn limit(&self) -> Result<Option<u64>, StoreError> {
237        self.limit
238            .get_or_try_init(|| async {
239                let viewer = self.client.viewer().await.map_err(store_error)?;
240                Ok::<_, StoreError>(viewer.server.max_item_bytes)
241            })
242            .await
243            .copied()
244    }
245
246    /// Settles an upload whose commit was lost: the id was ours, so the
247    /// item is there or it is not.
248    async fn settle(&self, id: &ItemId) -> Result<PutOutcome, StoreError> {
249        match self.item(id).await? {
250            Some(item) => Ok(PutOutcome {
251                meta: self.decode(&item)?.0,
252                created: true,
253            }),
254            None => Err(StoreError::Backend(format!(
255                "the upload of {id} was lost; send it again"
256            ))),
257        }
258    }
259
260    /// The id `meta`'s item gets in this store: its creation time, and this
261    /// store's content key for its SHA-256.
262    ///
263    /// # Errors
264    ///
265    /// [`StoreError::Corrupt`] when `meta`'s SHA-256 is not one.
266    pub fn id_for(&self, meta: &ItemMeta) -> Result<ItemId, StoreError> {
267        let sha256 = format::decode_array(&meta.sha256)
268            .map_err(|reason| corrupt(meta.id.as_str(), format!("sha256: {reason}")))?;
269        let key = self.content_key(&ContentDigest::new(sha256, meta.size));
270        Ok(ItemId::new(meta.created_at, key)?)
271    }
272
273    /// Stores another store's item, with its metadata and creation time,
274    /// under the id [`HttpStore::id_for`] gives it, and returns its
275    /// metadata here. An item already stored under that id is left as it
276    /// is. In the staged partition this is a re-encryption's copy.
277    ///
278    /// # Errors
279    ///
280    /// [`StoreError::Corrupt`] when the content does not match `meta`'s
281    /// SHA-256 and size, and the server's refusals.
282    pub async fn import(&self, meta: &ItemMeta, content: BoxRead) -> Result<ItemMeta, StoreError> {
283        let spool = tempfile::NamedTempFile::new().map_err(temp_error)?;
284        let mut file = tokio::fs::File::from_std(spool.reopen().map_err(temp_error)?);
285        let written = format::write_content(content, &mut file, None)
286            .await
287            .map_err(copy_error)?;
288        if written.digest.sha256_hex() != meta.sha256 || written.digest.size() != meta.size {
289            return Err(corrupt(
290                meta.id.as_str(),
291                "its content does not match its SHA-256",
292            ));
293        }
294        let copy = ItemMeta {
295            id: self.id_for(meta)?,
296            ..meta.clone()
297        };
298        Ok(self.upload(copy, spool).await?.meta)
299    }
300
301    /// Seals the content spooled in `spool` when the store is sealed, and
302    /// uploads it as `meta`'s item.
303    async fn upload(
304        &self,
305        meta: ItemMeta,
306        spool: tempfile::NamedTempFile,
307    ) -> Result<PutOutcome, StoreError> {
308        let (stored, salt) = match &self.sealer {
309            Some(sealer) => {
310                let sealing = sealer.content_sealer()?;
311                let salt = sealing.salt();
312                let sealed = tempfile::NamedTempFile::new().map_err(temp_error)?;
313                let mut out = tokio::fs::File::from_std(sealed.reopen().map_err(temp_error)?);
314                let input = tokio::fs::File::open(spool.path())
315                    .await
316                    .map_err(temp_error)?;
317                format::seal_content(input, &mut out, sealing)
318                    .await
319                    .map_err(copy_error)?;
320                (sealed, Some(salt))
321            }
322            None => (spool, None),
323        };
324        let size = stored.as_file().metadata().map_err(temp_error)?.len();
325        if let Some(limit) = self.limit().await?
326            && size > limit
327        {
328            return Err(StoreError::Backend(format!(
329                "the item takes {size} bytes on the server, over its limit of {limit} bytes for one item"
330            )));
331        }
332
333        let sealing = self.sealer.as_ref().zip(salt.as_ref());
334        let meta_json = format::encode_meta(&meta, sealing).map_err(|err| match err {
335            format::EncodeError::Crypto(err) => err.into(),
336            format::EncodeError::Json(err) => corrupt(meta.id.as_str(), err),
337        })?;
338        let text = String::from_utf8(meta_json).map_err(|err| corrupt(meta.id.as_str(), err))?;
339        // A JSON value cannot carry the file's final newline.
340        let raw = RawValue::from_string(text.trim_end().to_owned())
341            .map_err(|err| corrupt(meta.id.as_str(), err))?;
342        let request = serde_json::to_vec(&BeginUpload {
343            id: meta.id.as_str(),
344            meta: &raw,
345            size: size.to_string(),
346            expected_key_id: self.expected().await?,
347            in_rewrite: self.partition == Partition::Staged,
348        })
349        .map_err(|err| corrupt(meta.id.as_str(), err))?;
350        let uploads = self.url("/uploads", &[])?;
351        let response = match self
352            .client
353            .send("beginUpload", |http| {
354                http.post(uploads.clone())
355                    .header(CONTENT_TYPE, "application/json")
356                    .body(request.clone())
357            })
358            .await
359        {
360            Ok(response) => response,
361            Err(err) if err.code() == Some(Code::QuotaExceeded) => {
362                return Err(self.over_quota(&err, size).await);
363            }
364            Err(err) => return Err(store_error(err)),
365        };
366        if response.status() == StatusCode::OK {
367            // The content is stored already: nothing is sent.
368            let outcome: api::PutOutcome = json(response).await.map_err(store_error)?;
369            tracing::info!(id = %outcome.item.id, "item already present");
370            return Ok(PutOutcome {
371                meta: self.decode(&outcome.item)?.0,
372                created: false,
373            });
374        }
375        let ticket: UploadTicket = json(response).await.map_err(store_error)?;
376        match self
377            .send_and_commit(&ticket, stored.path(), size, &meta.id)
378            .await
379        {
380            Ok(outcome) => {
381                tracing::info!(id = %outcome.meta.id, size = outcome.meta.size, created = outcome.created, "item stored");
382                Ok(outcome)
383            }
384            Err(err) => {
385                self.abort(&ticket).await;
386                Err(err)
387            }
388        }
389    }
390
391    /// The refusal of an item that does not fit the workspace's quota,
392    /// naming the quota and what is used.
393    async fn over_quota(&self, err: &HttpsError, size: u64) -> StoreError {
394        match self.client.workspace().await {
395            Ok(workspace) => StoreError::Backend(format!(
396                "{err}: the item takes {size} bytes, and the workspace uses {} of its quota of {} bytes; delete or prune items, or ask the server's operator for a larger quota",
397                workspace.used_bytes, workspace.quota_bytes
398            )),
399            Err(_) => StoreError::Backend(err.to_string()),
400        }
401    }
402
403    /// Sends the stored bytes in `path` to the ticket, then commits it.
404    async fn send_and_commit(
405        &self,
406        ticket: &UploadTicket,
407        path: &std::path::Path,
408        size: u64,
409        id: &ItemId,
410    ) -> Result<PutOutcome, StoreError> {
411        let content = self.url(&format!("/uploads/{}/content", ticket.upload_id), &[])?;
412        self.client
413            .send("putUploadContent", |http| {
414                http.put(content.clone())
415                    .header(CONTENT_LENGTH, size)
416                    .body(file_body(path))
417            })
418            .await
419            .map_err(store_error)?;
420        let commit = self.url(&format!("/uploads/{}/commit", ticket.upload_id), &[])?;
421        match self
422            .client
423            .send("commitUpload", |http| http.post(commit.clone()))
424            .await
425        {
426            Ok(response) => {
427                let outcome: api::PutOutcome = json(response).await.map_err(store_error)?;
428                Ok(PutOutcome {
429                    meta: self.decode(&outcome.item)?.0,
430                    created: outcome.created,
431                })
432            }
433            Err(err) if is_not_found(&err) => self.settle(id).await,
434            Err(err) => Err(store_error(err)),
435        }
436    }
437
438    async fn abort(&self, ticket: &UploadTicket) {
439        let Ok(url) = self.url(&format!("/uploads/{}", ticket.upload_id), &[]) else {
440            return;
441        };
442        if let Err(err) = self
443            .client
444            .send("abortUpload", |http| http.delete(url.clone()))
445            .await
446        {
447            tracing::debug!(error = %err, "could not abort the upload; the server's janitor will");
448        }
449    }
450}
451
452/// A request body that streams the file at `path`.
453fn file_body(path: &std::path::Path) -> reqwest::Body {
454    match std::fs::File::open(path) {
455        Ok(file) => reqwest::Body::wrap_stream(tokio_util::io::ReaderStream::new(
456            tokio::fs::File::from_std(file),
457        )),
458        Err(err) => reqwest::Body::wrap_stream(futures_util::stream::iter([Err::<Bytes, _>(err)])),
459    }
460}
461
462fn copy_error(err: CopyError) -> StoreError {
463    match err {
464        CopyError::Read(err) => StoreError::Content(err.to_string()),
465        CopyError::Write(err) => StoreError::Backend(format!("writing a temporary file: {err}")),
466        CopyError::Crypto(err) => err.into(),
467    }
468}
469
470fn temp_error(err: io::Error) -> StoreError {
471    StoreError::Backend(format!("a temporary file: {err}"))
472}
473
474#[async_trait]
475impl Store for HttpStore {
476    fn key_id(&self) -> Option<KeyId> {
477        self.sealer.as_ref().map(Sealer::key_id)
478    }
479
480    fn content_key(&self, digest: &ContentDigest) -> ContentKey {
481        match &self.sealer {
482            Some(sealer) => sealer.content_key(digest),
483            None => digest.content_key(),
484        }
485    }
486
487    async fn put(&self, item: NewItem, content: BoxRead) -> Result<PutOutcome, StoreError> {
488        // The id and metadata come before the content, so the content is
489        // hashed first, into a file only this user may read.
490        let spool = tempfile::NamedTempFile::new().map_err(temp_error)?;
491        let mut file = tokio::fs::File::from_std(spool.reopen().map_err(temp_error)?);
492        let written = format::write_content(content, &mut file, None)
493            .await
494            .map_err(copy_error)?;
495        let key = self.content_key(&written.digest);
496        let preview = format::preview(item.kind, &written.head);
497        let meta = item.finish_keyed(self.clock.now(), &written.digest, key, preview)?;
498        self.upload(meta, spool).await
499    }
500
501    async fn list(&self) -> Result<Vec<ItemMeta>, StoreError> {
502        self.remind_plain_left().await;
503        self.list_after(None).await
504    }
505
506    async fn list_after(&self, after: Option<&ItemId>) -> Result<Vec<ItemMeta>, StoreError> {
507        let url = match after {
508            Some(after) => self.url("/items", &[("after", after.as_str())])?,
509            None => self.url("/items", &[])?,
510        };
511        let response = self
512            .client
513            .send("listItems", |http| http.get(url.clone()))
514            .await
515            .map_err(store_error)?;
516        let items: Vec<api::Item> = json(response).await.map_err(store_error)?;
517        let mut metas = Vec::with_capacity(items.len());
518        for item in &items {
519            match self.decode(item) {
520                Ok((meta, _)) => metas.push(meta),
521                Err(err) => tracing::warn!(id = %item.id, error = %err, "skipping corrupt item"),
522            }
523        }
524        Ok(metas)
525    }
526
527    async fn list_ids(&self) -> Result<Vec<ItemId>, StoreError> {
528        let url = self.url("/item-ids", &[])?;
529        let response = self
530            .client
531            .send("listItemIds", |http| http.get(url.clone()))
532            .await
533            .map_err(store_error)?;
534        let ids: Vec<String> = json(response).await.map_err(store_error)?;
535        Ok(ids.iter().filter_map(|id| ItemId::parse(id).ok()).collect())
536    }
537
538    async fn get(&self, id: &ItemId) -> Result<(ItemMeta, BoxRead), StoreError> {
539        let item = self
540            .item(id)
541            .await?
542            .ok_or_else(|| StoreError::NotFound(id.to_string()))?;
543        let (meta, salt) = self.decode(&item)?;
544        let url = self.url(&format!("/items/{id}/content"), &[])?;
545        let content = download(self.client.clone(), url, item.stored_bytes);
546        match (&self.sealer, salt) {
547            (Some(sealer), Some(salt)) => {
548                let expected = crypto::sealed_len(meta.size);
549                if item.stored_bytes != expected {
550                    return Err(corrupt(
551                        id.as_str(),
552                        format!("content has {} of {expected} bytes", item.stored_bytes),
553                    ));
554                }
555                Ok((meta, Box::new(sealer.open_item_content(content, salt))))
556            }
557            _ => Ok((meta, content)),
558        }
559    }
560
561    async fn get_meta(&self, id: &ItemId) -> Result<ItemMeta, StoreError> {
562        match self.item(id).await? {
563            Some(item) => Ok(self.decode(&item)?.0),
564            None => Err(StoreError::NotFound(id.to_string())),
565        }
566    }
567
568    async fn exists(&self, id: &ItemId) -> Result<bool, StoreError> {
569        Ok(self.item(id).await?.is_some())
570    }
571
572    async fn find_by_content_key(&self, key: &ContentKey) -> Result<Option<ItemMeta>, StoreError> {
573        let url = self.url(&format!("/content-keys/{key}"), &[])?;
574        match self
575            .client
576            .send("findByContentKey", |http| http.get(url.clone()))
577            .await
578        {
579            Ok(response) => {
580                let item: api::Item = json(response).await.map_err(store_error)?;
581                Ok(Some(self.decode(&item)?.0))
582            }
583            Err(err) if is_not_found(&err) => Ok(None),
584            Err(err) => Err(store_error(err)),
585        }
586    }
587
588    async fn resolve(&self, input: &str) -> Result<ItemId, StoreError> {
589        let url = self.url("/items/resolve", &[("input", input)])?;
590        let response = self
591            .client
592            .send("resolveItem", |http| http.get(url.clone()))
593            .await
594            .map_err(store_error)?;
595        let resolved: Resolved = json(response).await.map_err(store_error)?;
596        let parse = |id: &str| ItemId::parse(id).map_err(|err| corrupt(id, err));
597        match resolved.result.as_str() {
598            "resolved" => parse(resolved.id.as_deref().unwrap_or_default()),
599            "ambiguous" => Err(StoreError::Ambiguous {
600                input: input.trim().to_owned(),
601                candidates: resolved
602                    .candidates
603                    .iter()
604                    .map(|id| parse(id))
605                    .collect::<Result<_, _>>()?,
606            }),
607            "invalidPrefix" => Err(StoreError::InvalidPrefix(input.trim().to_owned())),
608            _ => Err(StoreError::NotFound(input.trim().to_owned())),
609        }
610    }
611
612    async fn delete(&self, id: &ItemId) -> Result<ItemMeta, StoreError> {
613        let expected = self.expected().await?;
614        let mut query = Vec::new();
615        if let Some(key) = &expected {
616            query.push(("expectedKeyId", key.as_str()));
617        }
618        let url = self.url(&format!("/items/{id}"), &query)?;
619        match self
620            .client
621            .send("deleteItem", |http| http.delete(url.clone()))
622            .await
623        {
624            Ok(response) => {
625                let item: api::Item = json(response).await.map_err(store_error)?;
626                Ok(self.decode(&item)?.0)
627            }
628            Err(err) if is_not_found(&err) => Err(StoreError::NotFound(id.to_string())),
629            Err(err) => Err(store_error(err)),
630        }
631    }
632
633    async fn clean_staging(&self, older_than: Duration) -> Result<usize, StoreError> {
634        let url = self.url("/workspace/clean-staging", &[])?;
635        let body = serde_json::json!({ "olderThanSecs": older_than.as_secs() }).to_string();
636        let response = self
637            .client
638            .send("cleanStaging", |http| {
639                http.post(url.clone())
640                    .header(CONTENT_TYPE, "application/json")
641                    .body(body.clone())
642            })
643            .await
644            .map_err(store_error)?;
645        let cleaned: Cleaned = json(response).await.map_err(store_error)?;
646        Ok(cleaned.removed)
647    }
648
649    async fn probe_write(&self) -> Result<WriteProbe, StoreError> {
650        let url = self.url("/workspace/probe", &[])?;
651        self.client
652            .send("probeWrite", |http| http.post(url.clone()))
653            .await
654            .map_err(store_error)?;
655        Ok(WriteProbe::Verified)
656    }
657}
658
659/// An item's content, read from `url`. A read that breaks off before
660/// `total` bytes is resumed with `Range` from the last byte received, at
661/// most [`MAX_RESUMES`] times; the reader's caller checks the whole, by its
662/// SHA-256 or its seal.
663fn download(client: Arc<Client>, url: Url, total: u64) -> BoxRead {
664    let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, io::Error>>(8);
665    tokio::spawn(async move {
666        let mut offset = 0_u64;
667        let mut resumes = 0;
668        loop {
669            let range = (offset > 0).then(|| format!("bytes={offset}-"));
670            let response = client
671                .send("getItemContent", |http| {
672                    let request = http.get(url.clone());
673                    match &range {
674                        Some(range) => request.header(RANGE, range),
675                        None => request,
676                    }
677                })
678                .await;
679            let mut response = match response {
680                Ok(response) => response,
681                Err(err) => {
682                    let _ = tx.send(Err(io::Error::other(err.to_string()))).await;
683                    return;
684                }
685            };
686            if offset > 0 && response.status() != StatusCode::PARTIAL_CONTENT {
687                let _ = tx
688                    .send(Err(io::Error::other(
689                        "the server did not resume the download where it broke off",
690                    )))
691                    .await;
692                return;
693            }
694            let broke = loop {
695                match response.chunk().await {
696                    Ok(Some(chunk)) => {
697                        offset += chunk.len() as u64;
698                        if tx.send(Ok(chunk)).await.is_err() {
699                            return;
700                        }
701                    }
702                    Ok(None) => break None,
703                    Err(err) => break Some(err.to_string()),
704                }
705            };
706            if broke.is_none() && offset >= total {
707                return;
708            }
709            resumes += 1;
710            if resumes > MAX_RESUMES {
711                let reason = broke.unwrap_or_else(|| "it ended early".to_owned());
712                let _ = tx
713                    .send(Err(io::Error::other(format!(
714                        "the download broke off at {offset} of {total} bytes: {reason}"
715                    ))))
716                    .await;
717                return;
718            }
719            tracing::info!(offset, total, "download broke off; resuming");
720        }
721    });
722    let stream = Box::pin(futures_util::stream::unfold(rx, |mut rx| async move {
723        rx.recv().await.map(|item| (item, rx))
724    }));
725    Box::new(tokio_util::io::StreamReader::new(stream))
726}