1use std::io;
12use std::sync::Arc;
13use std::time::Duration;
14
15use async_trait::async_trait;
16use bytes::Bytes;
17use passalong_core::clock::Clock;
18use passalong_core::crypto::{self, KeyId, Sealer};
19use passalong_core::encryption::EncryptionError;
20use passalong_core::fs::BoxRead;
21use passalong_core::model::{ContentDigest, ContentKey, ItemId, ItemMeta, NewItem};
22use passalong_core::store::format::{self, CopyError, MetaError};
23use passalong_core::store::{PutOutcome, Store, StoreError, WriteProbe};
24use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, RANGE};
25use reqwest::{StatusCode, Url};
26use serde_json::value::RawValue;
27use tokio::sync::OnceCell;
28
29use crate::api::{self, BeginUpload, Cleaned, Resolved, UploadTicket};
30use crate::client::{Client, json};
31use crate::error::{Code, HttpsError};
32
33pub const MAX_RESUMES: u32 = 3;
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38#[non_exhaustive]
39pub enum Partition {
40 Current,
42 Plain,
44 Staged,
46}
47
48impl Partition {
49 fn name(self) -> &'static str {
50 match self {
51 Self::Current => "current",
52 Self::Plain => "plain",
53 Self::Staged => "staged",
54 }
55 }
56}
57
58pub struct HttpStore {
60 client: Arc<Client>,
61 sealer: Option<Sealer>,
62 clock: Arc<dyn Clock>,
63 partition: Partition,
64 limit: OnceCell<Option<u64>>,
66 workspace_key: OnceCell<Option<String>>,
68}
69
70impl std::fmt::Debug for HttpStore {
71 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72 f.debug_struct("HttpStore")
73 .field("client", &self.client)
74 .field("sealed", &self.sealer.is_some())
75 .field("partition", &self.partition)
76 .finish_non_exhaustive()
77 }
78}
79
80pub fn store_error(err: HttpsError) -> StoreError {
84 match err.code() {
85 Some(Code::KeyIdMismatch) => EncryptionError::KeyChanged.into(),
86 Some(Code::RewriteInProgress) => EncryptionError::Rewriting { started: None }.into(),
87 Some(Code::Unauthenticated | Code::KeyExpired | Code::KeyRevoked) => StoreError::Denied(
88 format!("{err}; ask the server's operator for a new API key"),
89 ),
90 _ => StoreError::Backend(err.to_string()),
91 }
92}
93
94fn is_not_found(err: &HttpsError) -> bool {
95 err.code() == Some(Code::NotFound)
96}
97
98fn corrupt(id: &str, reason: impl std::fmt::Display) -> StoreError {
99 StoreError::Corrupt {
100 id: id.to_owned(),
101 reason: reason.to_string(),
102 }
103}
104
105impl HttpStore {
106 pub fn new(client: Arc<Client>, sealer: Option<Sealer>, clock: Arc<dyn Clock>) -> Self {
108 Self::in_partition(client, sealer, clock, Partition::Current)
109 }
110
111 pub fn in_partition(
113 client: Arc<Client>,
114 sealer: Option<Sealer>,
115 clock: Arc<dyn Clock>,
116 partition: Partition,
117 ) -> Self {
118 Self {
119 client,
120 sealer,
121 clock,
122 partition,
123 limit: OnceCell::new(),
124 workspace_key: OnceCell::new(),
125 }
126 }
127
128 pub fn client(&self) -> &Arc<Client> {
130 &self.client
131 }
132
133 fn url(&self, path: &str, query: &[(&str, &str)]) -> Result<Url, StoreError> {
134 let mut url = Url::parse(&self.client.url(path))
135 .map_err(|err| StoreError::Config(format!("server.https.url: {err}")))?;
136 let item_route = path == "/items"
139 || path == "/item-ids"
140 || (path.starts_with("/items/") && path != "/items/resolve");
141 let partition =
142 (self.partition != Partition::Current && item_route).then(|| self.partition.name());
143 if partition.is_some() || !query.is_empty() {
144 let mut pairs = url.query_pairs_mut();
145 if let Some(partition) = partition {
146 pairs.append_pair("partition", partition);
147 }
148 for (name, value) in query {
149 pairs.append_pair(name, value);
150 }
151 }
152 Ok(url)
153 }
154
155 async fn expected(&self) -> Result<Option<String>, StoreError> {
159 if self.partition != Partition::Plain {
160 return Ok(self
161 .sealer
162 .as_ref()
163 .map(|sealer| sealer.key_id().to_string()));
164 }
165 self.workspace_key
166 .get_or_try_init(|| async {
167 let workspace = self.client.workspace().await.map_err(store_error)?;
168 Ok::<_, StoreError>(workspace.encryption.key_id)
169 })
170 .await
171 .cloned()
172 }
173
174 async fn remind_plain_left(&self) {
177 if self.sealer.is_none() || self.partition != Partition::Current {
178 return;
179 }
180 let Ok(mut url) = Url::parse(&self.client.url("/item-ids")) else {
181 return;
182 };
183 url.query_pairs_mut().append_pair("partition", "plain");
184 let Ok(response) = self
185 .client
186 .send("listItemIds", |http| http.get(url.clone()))
187 .await
188 else {
189 return;
190 };
191 let n = json::<Vec<String>>(response)
192 .await
193 .map_or(0, |ids| ids.len());
194 if n > 0 {
195 let (items, them) = if n == 1 {
196 ("item remains", "it")
197 } else {
198 ("items remain", "them")
199 };
200 tracing::warn!(
201 "{n} unencrypted {items} from before encryption; remove {them} with `passalong prune --plain`"
202 );
203 }
204 }
205
206 fn decode(
208 &self,
209 item: &api::Item,
210 ) -> Result<(ItemMeta, Option<format::ContentSalt>), StoreError> {
211 let id = ItemId::parse(&item.id).map_err(|err| corrupt(&item.id, err))?;
212 format::decode_meta(&id, item.meta.get().as_bytes(), self.sealer.as_ref()).map_err(|err| {
213 match err {
214 MetaError::Corrupt(reason) | MetaError::Unopened(reason) => {
217 corrupt(&item.id, reason)
218 }
219 }
220 })
221 }
222
223 async fn item(&self, id: &ItemId) -> Result<Option<api::Item>, StoreError> {
224 let url = self.url(&format!("/items/{id}"), &[])?;
225 match self
226 .client
227 .send("getItem", |http| http.get(url.clone()))
228 .await
229 {
230 Ok(response) => Ok(Some(json(response).await.map_err(store_error)?)),
231 Err(err) if is_not_found(&err) => Ok(None),
232 Err(err) => Err(store_error(err)),
233 }
234 }
235
236 async fn limit(&self) -> Result<Option<u64>, StoreError> {
237 self.limit
238 .get_or_try_init(|| async {
239 let viewer = self.client.viewer().await.map_err(store_error)?;
240 Ok::<_, StoreError>(viewer.server.max_item_bytes)
241 })
242 .await
243 .copied()
244 }
245
246 async fn settle(&self, id: &ItemId) -> Result<PutOutcome, StoreError> {
249 match self.item(id).await? {
250 Some(item) => Ok(PutOutcome {
251 meta: self.decode(&item)?.0,
252 created: true,
253 }),
254 None => Err(StoreError::Backend(format!(
255 "the upload of {id} was lost; send it again"
256 ))),
257 }
258 }
259
260 pub fn id_for(&self, meta: &ItemMeta) -> Result<ItemId, StoreError> {
267 let sha256 = format::decode_array(&meta.sha256)
268 .map_err(|reason| corrupt(meta.id.as_str(), format!("sha256: {reason}")))?;
269 let key = self.content_key(&ContentDigest::new(sha256, meta.size));
270 Ok(ItemId::new(meta.created_at, key)?)
271 }
272
273 pub async fn import(&self, meta: &ItemMeta, content: BoxRead) -> Result<ItemMeta, StoreError> {
283 let spool = tempfile::NamedTempFile::new().map_err(temp_error)?;
284 let mut file = tokio::fs::File::from_std(spool.reopen().map_err(temp_error)?);
285 let written = format::write_content(content, &mut file, None)
286 .await
287 .map_err(copy_error)?;
288 if written.digest.sha256_hex() != meta.sha256 || written.digest.size() != meta.size {
289 return Err(corrupt(
290 meta.id.as_str(),
291 "its content does not match its SHA-256",
292 ));
293 }
294 let copy = ItemMeta {
295 id: self.id_for(meta)?,
296 ..meta.clone()
297 };
298 Ok(self.upload(copy, spool).await?.meta)
299 }
300
301 async fn upload(
304 &self,
305 meta: ItemMeta,
306 spool: tempfile::NamedTempFile,
307 ) -> Result<PutOutcome, StoreError> {
308 let (stored, salt) = match &self.sealer {
309 Some(sealer) => {
310 let sealing = sealer.content_sealer()?;
311 let salt = sealing.salt();
312 let sealed = tempfile::NamedTempFile::new().map_err(temp_error)?;
313 let mut out = tokio::fs::File::from_std(sealed.reopen().map_err(temp_error)?);
314 let input = tokio::fs::File::open(spool.path())
315 .await
316 .map_err(temp_error)?;
317 format::seal_content(input, &mut out, sealing)
318 .await
319 .map_err(copy_error)?;
320 (sealed, Some(salt))
321 }
322 None => (spool, None),
323 };
324 let size = stored.as_file().metadata().map_err(temp_error)?.len();
325 if let Some(limit) = self.limit().await?
326 && size > limit
327 {
328 return Err(StoreError::Backend(format!(
329 "the item takes {size} bytes on the server, over its limit of {limit} bytes for one item"
330 )));
331 }
332
333 let sealing = self.sealer.as_ref().zip(salt.as_ref());
334 let meta_json = format::encode_meta(&meta, sealing).map_err(|err| match err {
335 format::EncodeError::Crypto(err) => err.into(),
336 format::EncodeError::Json(err) => corrupt(meta.id.as_str(), err),
337 })?;
338 let text = String::from_utf8(meta_json).map_err(|err| corrupt(meta.id.as_str(), err))?;
339 let raw = RawValue::from_string(text.trim_end().to_owned())
341 .map_err(|err| corrupt(meta.id.as_str(), err))?;
342 let request = serde_json::to_vec(&BeginUpload {
343 id: meta.id.as_str(),
344 meta: &raw,
345 size: size.to_string(),
346 expected_key_id: self.expected().await?,
347 in_rewrite: self.partition == Partition::Staged,
348 })
349 .map_err(|err| corrupt(meta.id.as_str(), err))?;
350 let uploads = self.url("/uploads", &[])?;
351 let response = match self
352 .client
353 .send("beginUpload", |http| {
354 http.post(uploads.clone())
355 .header(CONTENT_TYPE, "application/json")
356 .body(request.clone())
357 })
358 .await
359 {
360 Ok(response) => response,
361 Err(err) if err.code() == Some(Code::QuotaExceeded) => {
362 return Err(self.over_quota(&err, size).await);
363 }
364 Err(err) => return Err(store_error(err)),
365 };
366 if response.status() == StatusCode::OK {
367 let outcome: api::PutOutcome = json(response).await.map_err(store_error)?;
369 tracing::info!(id = %outcome.item.id, "item already present");
370 return Ok(PutOutcome {
371 meta: self.decode(&outcome.item)?.0,
372 created: false,
373 });
374 }
375 let ticket: UploadTicket = json(response).await.map_err(store_error)?;
376 match self
377 .send_and_commit(&ticket, stored.path(), size, &meta.id)
378 .await
379 {
380 Ok(outcome) => {
381 tracing::info!(id = %outcome.meta.id, size = outcome.meta.size, created = outcome.created, "item stored");
382 Ok(outcome)
383 }
384 Err(err) => {
385 self.abort(&ticket).await;
386 Err(err)
387 }
388 }
389 }
390
391 async fn over_quota(&self, err: &HttpsError, size: u64) -> StoreError {
394 match self.client.workspace().await {
395 Ok(workspace) => StoreError::Backend(format!(
396 "{err}: the item takes {size} bytes, and the workspace uses {} of its quota of {} bytes; delete or prune items, or ask the server's operator for a larger quota",
397 workspace.used_bytes, workspace.quota_bytes
398 )),
399 Err(_) => StoreError::Backend(err.to_string()),
400 }
401 }
402
403 async fn send_and_commit(
405 &self,
406 ticket: &UploadTicket,
407 path: &std::path::Path,
408 size: u64,
409 id: &ItemId,
410 ) -> Result<PutOutcome, StoreError> {
411 let content = self.url(&format!("/uploads/{}/content", ticket.upload_id), &[])?;
412 self.client
413 .send("putUploadContent", |http| {
414 http.put(content.clone())
415 .header(CONTENT_LENGTH, size)
416 .body(file_body(path))
417 })
418 .await
419 .map_err(store_error)?;
420 let commit = self.url(&format!("/uploads/{}/commit", ticket.upload_id), &[])?;
421 match self
422 .client
423 .send("commitUpload", |http| http.post(commit.clone()))
424 .await
425 {
426 Ok(response) => {
427 let outcome: api::PutOutcome = json(response).await.map_err(store_error)?;
428 Ok(PutOutcome {
429 meta: self.decode(&outcome.item)?.0,
430 created: outcome.created,
431 })
432 }
433 Err(err) if is_not_found(&err) => self.settle(id).await,
434 Err(err) => Err(store_error(err)),
435 }
436 }
437
438 async fn abort(&self, ticket: &UploadTicket) {
439 let Ok(url) = self.url(&format!("/uploads/{}", ticket.upload_id), &[]) else {
440 return;
441 };
442 if let Err(err) = self
443 .client
444 .send("abortUpload", |http| http.delete(url.clone()))
445 .await
446 {
447 tracing::debug!(error = %err, "could not abort the upload; the server's janitor will");
448 }
449 }
450}
451
452fn file_body(path: &std::path::Path) -> reqwest::Body {
454 match std::fs::File::open(path) {
455 Ok(file) => reqwest::Body::wrap_stream(tokio_util::io::ReaderStream::new(
456 tokio::fs::File::from_std(file),
457 )),
458 Err(err) => reqwest::Body::wrap_stream(futures_util::stream::iter([Err::<Bytes, _>(err)])),
459 }
460}
461
462fn copy_error(err: CopyError) -> StoreError {
463 match err {
464 CopyError::Read(err) => StoreError::Content(err.to_string()),
465 CopyError::Write(err) => StoreError::Backend(format!("writing a temporary file: {err}")),
466 CopyError::Crypto(err) => err.into(),
467 }
468}
469
470fn temp_error(err: io::Error) -> StoreError {
471 StoreError::Backend(format!("a temporary file: {err}"))
472}
473
474#[async_trait]
475impl Store for HttpStore {
476 fn key_id(&self) -> Option<KeyId> {
477 self.sealer.as_ref().map(Sealer::key_id)
478 }
479
480 fn content_key(&self, digest: &ContentDigest) -> ContentKey {
481 match &self.sealer {
482 Some(sealer) => sealer.content_key(digest),
483 None => digest.content_key(),
484 }
485 }
486
487 async fn put(&self, item: NewItem, content: BoxRead) -> Result<PutOutcome, StoreError> {
488 let spool = tempfile::NamedTempFile::new().map_err(temp_error)?;
491 let mut file = tokio::fs::File::from_std(spool.reopen().map_err(temp_error)?);
492 let written = format::write_content(content, &mut file, None)
493 .await
494 .map_err(copy_error)?;
495 let key = self.content_key(&written.digest);
496 let preview = format::preview(item.kind, &written.head);
497 let meta = item.finish_keyed(self.clock.now(), &written.digest, key, preview)?;
498 self.upload(meta, spool).await
499 }
500
501 async fn list(&self) -> Result<Vec<ItemMeta>, StoreError> {
502 self.remind_plain_left().await;
503 self.list_after(None).await
504 }
505
506 async fn list_after(&self, after: Option<&ItemId>) -> Result<Vec<ItemMeta>, StoreError> {
507 let url = match after {
508 Some(after) => self.url("/items", &[("after", after.as_str())])?,
509 None => self.url("/items", &[])?,
510 };
511 let response = self
512 .client
513 .send("listItems", |http| http.get(url.clone()))
514 .await
515 .map_err(store_error)?;
516 let items: Vec<api::Item> = json(response).await.map_err(store_error)?;
517 let mut metas = Vec::with_capacity(items.len());
518 for item in &items {
519 match self.decode(item) {
520 Ok((meta, _)) => metas.push(meta),
521 Err(err) => tracing::warn!(id = %item.id, error = %err, "skipping corrupt item"),
522 }
523 }
524 Ok(metas)
525 }
526
527 async fn list_ids(&self) -> Result<Vec<ItemId>, StoreError> {
528 let url = self.url("/item-ids", &[])?;
529 let response = self
530 .client
531 .send("listItemIds", |http| http.get(url.clone()))
532 .await
533 .map_err(store_error)?;
534 let ids: Vec<String> = json(response).await.map_err(store_error)?;
535 Ok(ids.iter().filter_map(|id| ItemId::parse(id).ok()).collect())
536 }
537
538 async fn get(&self, id: &ItemId) -> Result<(ItemMeta, BoxRead), StoreError> {
539 let item = self
540 .item(id)
541 .await?
542 .ok_or_else(|| StoreError::NotFound(id.to_string()))?;
543 let (meta, salt) = self.decode(&item)?;
544 let url = self.url(&format!("/items/{id}/content"), &[])?;
545 let content = download(self.client.clone(), url, item.stored_bytes);
546 match (&self.sealer, salt) {
547 (Some(sealer), Some(salt)) => {
548 let expected = crypto::sealed_len(meta.size);
549 if item.stored_bytes != expected {
550 return Err(corrupt(
551 id.as_str(),
552 format!("content has {} of {expected} bytes", item.stored_bytes),
553 ));
554 }
555 Ok((meta, Box::new(sealer.open_item_content(content, salt))))
556 }
557 _ => Ok((meta, content)),
558 }
559 }
560
561 async fn get_meta(&self, id: &ItemId) -> Result<ItemMeta, StoreError> {
562 match self.item(id).await? {
563 Some(item) => Ok(self.decode(&item)?.0),
564 None => Err(StoreError::NotFound(id.to_string())),
565 }
566 }
567
568 async fn exists(&self, id: &ItemId) -> Result<bool, StoreError> {
569 Ok(self.item(id).await?.is_some())
570 }
571
572 async fn find_by_content_key(&self, key: &ContentKey) -> Result<Option<ItemMeta>, StoreError> {
573 let url = self.url(&format!("/content-keys/{key}"), &[])?;
574 match self
575 .client
576 .send("findByContentKey", |http| http.get(url.clone()))
577 .await
578 {
579 Ok(response) => {
580 let item: api::Item = json(response).await.map_err(store_error)?;
581 Ok(Some(self.decode(&item)?.0))
582 }
583 Err(err) if is_not_found(&err) => Ok(None),
584 Err(err) => Err(store_error(err)),
585 }
586 }
587
588 async fn resolve(&self, input: &str) -> Result<ItemId, StoreError> {
589 let url = self.url("/items/resolve", &[("input", input)])?;
590 let response = self
591 .client
592 .send("resolveItem", |http| http.get(url.clone()))
593 .await
594 .map_err(store_error)?;
595 let resolved: Resolved = json(response).await.map_err(store_error)?;
596 let parse = |id: &str| ItemId::parse(id).map_err(|err| corrupt(id, err));
597 match resolved.result.as_str() {
598 "resolved" => parse(resolved.id.as_deref().unwrap_or_default()),
599 "ambiguous" => Err(StoreError::Ambiguous {
600 input: input.trim().to_owned(),
601 candidates: resolved
602 .candidates
603 .iter()
604 .map(|id| parse(id))
605 .collect::<Result<_, _>>()?,
606 }),
607 "invalidPrefix" => Err(StoreError::InvalidPrefix(input.trim().to_owned())),
608 _ => Err(StoreError::NotFound(input.trim().to_owned())),
609 }
610 }
611
612 async fn delete(&self, id: &ItemId) -> Result<ItemMeta, StoreError> {
613 let expected = self.expected().await?;
614 let mut query = Vec::new();
615 if let Some(key) = &expected {
616 query.push(("expectedKeyId", key.as_str()));
617 }
618 let url = self.url(&format!("/items/{id}"), &query)?;
619 match self
620 .client
621 .send("deleteItem", |http| http.delete(url.clone()))
622 .await
623 {
624 Ok(response) => {
625 let item: api::Item = json(response).await.map_err(store_error)?;
626 Ok(self.decode(&item)?.0)
627 }
628 Err(err) if is_not_found(&err) => Err(StoreError::NotFound(id.to_string())),
629 Err(err) => Err(store_error(err)),
630 }
631 }
632
633 async fn clean_staging(&self, older_than: Duration) -> Result<usize, StoreError> {
634 let url = self.url("/workspace/clean-staging", &[])?;
635 let body = serde_json::json!({ "olderThanSecs": older_than.as_secs() }).to_string();
636 let response = self
637 .client
638 .send("cleanStaging", |http| {
639 http.post(url.clone())
640 .header(CONTENT_TYPE, "application/json")
641 .body(body.clone())
642 })
643 .await
644 .map_err(store_error)?;
645 let cleaned: Cleaned = json(response).await.map_err(store_error)?;
646 Ok(cleaned.removed)
647 }
648
649 async fn probe_write(&self) -> Result<WriteProbe, StoreError> {
650 let url = self.url("/workspace/probe", &[])?;
651 self.client
652 .send("probeWrite", |http| http.post(url.clone()))
653 .await
654 .map_err(store_error)?;
655 Ok(WriteProbe::Verified)
656 }
657}
658
659fn download(client: Arc<Client>, url: Url, total: u64) -> BoxRead {
664 let (tx, rx) = tokio::sync::mpsc::channel::<Result<Bytes, io::Error>>(8);
665 tokio::spawn(async move {
666 let mut offset = 0_u64;
667 let mut resumes = 0;
668 loop {
669 let range = (offset > 0).then(|| format!("bytes={offset}-"));
670 let response = client
671 .send("getItemContent", |http| {
672 let request = http.get(url.clone());
673 match &range {
674 Some(range) => request.header(RANGE, range),
675 None => request,
676 }
677 })
678 .await;
679 let mut response = match response {
680 Ok(response) => response,
681 Err(err) => {
682 let _ = tx.send(Err(io::Error::other(err.to_string()))).await;
683 return;
684 }
685 };
686 if offset > 0 && response.status() != StatusCode::PARTIAL_CONTENT {
687 let _ = tx
688 .send(Err(io::Error::other(
689 "the server did not resume the download where it broke off",
690 )))
691 .await;
692 return;
693 }
694 let broke = loop {
695 match response.chunk().await {
696 Ok(Some(chunk)) => {
697 offset += chunk.len() as u64;
698 if tx.send(Ok(chunk)).await.is_err() {
699 return;
700 }
701 }
702 Ok(None) => break None,
703 Err(err) => break Some(err.to_string()),
704 }
705 };
706 if broke.is_none() && offset >= total {
707 return;
708 }
709 resumes += 1;
710 if resumes > MAX_RESUMES {
711 let reason = broke.unwrap_or_else(|| "it ended early".to_owned());
712 let _ = tx
713 .send(Err(io::Error::other(format!(
714 "the download broke off at {offset} of {total} bytes: {reason}"
715 ))))
716 .await;
717 return;
718 }
719 tracing::info!(offset, total, "download broke off; resuming");
720 }
721 });
722 let stream = Box::pin(futures_util::stream::unfold(rx, |mut rx| async move {
723 rx.recv().await.map(|item| (item, rx))
724 }));
725 Box::new(tokio_util::io::StreamReader::new(stream))
726}