1use std::collections::HashSet;
8use std::path::Path;
9use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
10
11use anyhow::{Context, Result, bail, ensure};
12use serde::Serialize;
13use zeroize::Zeroizing;
14
15use super::errors::{code, has_code, is_access_refusal};
16use super::exchange::Link;
17use super::host::{ClientHost, key_store};
18use super::keys::peer::InvitationProgress;
19use super::keys::{
20 EnrollmentReadiness, ProtectedLocalKeyStore, ProtectedLocalKeyStoreError,
21 ProtectedLocalKeyStoreErrorKind,
22};
23use super::tail::{ImageTransfer, Round};
24use super::{DeviceInvitation, SetupInvitation, bootstrap, coordination, enrollment, tail};
25use crate::db::Database;
26use crate::sync::seed_claim::{ClaimAuthentication, membership::Invitation};
27
28mod devices;
29pub use devices::{
30 Device, DeviceListing, Removal, finish_removal, load_devices, remove_other_device,
31};
32mod reset;
33pub use reset::{Reset, reset};
34
35pub const ROUND_LIMIT: usize = 1000;
39const IMAGE_RETRY_ROUNDS: usize = 16;
42#[cfg(not(any(test, feature = "test-support")))]
43fn invitation_seconds() -> u64 {
44 600
45}
46#[cfg(any(test, feature = "test-support"))]
48fn invitation_seconds() -> u64 {
49 std::env::var("AVEN_TEST_INVITATION_SECONDS")
50 .ok()
51 .and_then(|value| value.parse().ok())
52 .unwrap_or(600)
53}
54#[cfg(not(any(test, feature = "test-support")))]
55const INVITER_POLL_INTERVAL: Duration = Duration::from_secs(1);
56#[cfg(any(test, feature = "test-support"))]
57const INVITER_POLL_INTERVAL: Duration = Duration::from_millis(100);
58#[cfg(not(any(test, feature = "test-support")))]
61const JOIN_POLL_INTERVALS: [Duration; 5] = [
62 Duration::from_millis(250),
63 Duration::from_millis(500),
64 Duration::from_millis(500),
65 Duration::from_secs(1),
66 Duration::from_secs(2),
67];
68#[cfg(any(test, feature = "test-support"))]
69const JOIN_POLL_INTERVALS: [Duration; 4] = [
70 Duration::from_millis(25),
71 Duration::from_millis(50),
72 Duration::from_millis(75),
73 Duration::from_millis(100),
74];
75
76fn join_poll_interval(attempt: usize) -> Duration {
77 jittered(JOIN_POLL_INTERVALS[attempt.min(JOIN_POLL_INTERVALS.len() - 1)])
78}
79
80#[cfg(not(any(test, feature = "test-support")))]
81fn jittered(interval: Duration) -> Duration {
82 let millis = u64::try_from(interval.as_millis()).unwrap_or(u64::MAX);
83 let spread = millis / 10;
84 let mut random = [0_u8; 2];
85 if getrandom::fill(&mut random).is_err() {
86 return interval;
87 }
88 let offset = u64::from(u16::from_le_bytes(random)) % (spread * 2 + 1);
89 Duration::from_millis(millis - spread + offset)
90}
91
92#[cfg(any(test, feature = "test-support"))]
93fn jittered(interval: Duration) -> Duration {
94 interval
95}
96
97pub const NOT_SET_UP: &str = "error sync-not-set-up hint=\"run `aven sync setup` with an invitation from your hosting provider or `aven server setup`, or `aven sync join` on a new database\"";
98
99pub async fn is_set_up(database: &Database) -> Result<bool> {
102 Ok(database.enrollment_pin().await?.is_some() || has_seed_setup(database).await?)
103}
104
105async fn has_seed_setup(database: &Database) -> Result<bool> {
108 Ok(database.local_seed_genesis_commitment().await?.is_some()
109 && !database.local_seed_claim_refused().await?)
110}
111
112#[derive(Clone, Copy, Debug, PartialEq, Eq)]
115pub enum LocalPhase {
116 NotSetUp,
117 SetupIncomplete,
119 JoinIncomplete,
121 SetUp,
122}
123
124pub async fn local_phase(database: &Database) -> Result<LocalPhase> {
125 Ok(match database.enrollment_pin().await? {
126 Some((_, _, role)) if role == "peer" => {
127 if database.meta("e2ee_association").await?.is_some() {
128 LocalPhase::SetUp
129 } else {
130 LocalPhase::JoinIncomplete
131 }
132 }
133 Some(_) => LocalPhase::SetUp,
134 None if has_seed_setup(database).await? => LocalPhase::SetupIncomplete,
135 None => LocalPhase::NotSetUp,
136 })
137}
138
139#[derive(Clone, Copy, Debug, PartialEq, Eq)]
143pub struct Progress {
144 pub stage: Stage,
145 pub amount: Option<Amount>,
146}
147
148impl From<Stage> for Progress {
149 fn from(stage: Stage) -> Self {
150 Self {
151 stage,
152 amount: None,
153 }
154 }
155}
156
157#[derive(Clone, Copy, Debug, PartialEq, Eq)]
159pub enum Amount {
160 Bytes { done: u64, total: Option<u64> },
163 Changes { done: u64 },
165 Images { done: u64, remaining: u64 },
168}
169
170#[derive(Clone, Copy, Debug, PartialEq, Eq)]
173pub enum Stage {
174 PreparingData,
175 UploadingData,
176 FinishingSetup,
177 WaitingForInviter,
178 DownloadingTasks,
179 CatchingUp,
181 DownloadingImages,
183}
184
185pub fn unix_now() -> Result<u64> {
186 Ok(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs())
187}
188
189#[derive(Clone, Debug, PartialEq, Eq)]
191pub struct SetupPreview {
192 pub workspaces: usize,
193 pub tasks: i64,
194 pub missing_images: u64,
196 pub leaves_unencrypted_server: bool,
198}
199
200pub async fn setup_preview(database: &Database, host: &dyn ClientHost) -> Result<SetupPreview> {
201 let workspaces = database.list_workspaces().await?;
202 let blob_dir = host.blob_dir(database)?;
203 let mut tasks = 0;
204 for workspace in &workspaces {
205 tasks += database.workspace_task_counts(&workspace.id).await?.visible;
206 }
207 Ok(SetupPreview {
208 workspaces: workspaces.len(),
209 tasks,
210 missing_images: database
211 .missing_setup_attachment_counts(&blob_dir)
212 .await?
213 .count,
214 leaves_unencrypted_server: database.meta("sync_server_url").await?.is_some(),
215 })
216}
217
218pub async fn ensure_setup_available(database: &Database, host: &dyn ClientHost) -> Result<()> {
220 host.ensure_sync_allowed()?;
221 ensure!(
222 database.enrollment_pin().await?.is_none(),
223 "error sync-already-set-up hint=\"run `aven sync` or `aven sync status`\""
224 );
225 Ok(())
226}
227
228fn explain_seed_claim_error(error: anyhow::Error) -> anyhow::Error {
229 let setup_mismatch = error.chain().any(|cause| {
230 cause
231 .downcast_ref::<ProtectedLocalKeyStoreError>()
232 .is_some_and(|error| error.kind() == ProtectedLocalKeyStoreErrorKind::SetupMismatch)
233 });
234 if setup_mismatch {
235 error.context(
236 "error sync-setup-invitation-mismatch hint=\"resume with the invitation that started setup\"",
237 )
238 } else {
239 error
240 }
241}
242
243pub async fn run_setup(
247 link: Link,
248 database: &Database,
249 host: &dyn ClientHost,
250 invitation: &SetupInvitation,
251 progress: &(dyn Fn(Progress) + Sync),
252) -> Result<Outcome> {
253 Ok(setup(link, database, host, Some(invitation), progress)
254 .await?
255 .1)
256}
257
258pub const SETUP_INVITATION_REQUIRED: &str = "sync-setup-resume-invitation-required";
262
263pub async fn resume_setup(
269 link: Link,
270 database: &Database,
271 host: &dyn ClientHost,
272 progress: &(dyn Fn(Progress) + Sync),
273) -> Result<(String, Outcome)> {
274 setup(link, database, host, None, progress).await
275}
276
277async fn setup(
278 link: Link,
279 database: &Database,
280 host: &dyn ClientHost,
281 invitation: Option<&SetupInvitation>,
282 progress: &(dyn Fn(Progress) + Sync),
283) -> Result<(String, Outcome)> {
284 let _total = bootstrap::StageTimer::start("setup_total");
285 ensure_setup_available(database, host).await?;
286 let blob_dir = host.blob_dir(database)?;
287 let store = {
288 let _timer = bootstrap::StageTimer::start("key_store");
289 key_store(host, database).await?
290 };
291 let _guard = coordination::acquire(database).await?;
292 let (server, setup_id, secret) = match invitation {
293 Some(invitation) => {
294 store.bind_seed_origin(database, &invitation.server).await?;
295 (
296 invitation.server.clone(),
297 invitation.setup_id,
298 Some(&invitation.secret),
299 )
300 }
301 None => {
302 ensure!(
303 local_phase(database).await? == LocalPhase::SetupIncomplete,
304 "error sync-setup-not-started hint=\"rerun `aven sync setup`\""
305 );
306 let (server, setup_id) = store.retained_seed_setup(database).await?;
307 (server, setup_id, None)
308 }
309 };
310 let bootstrap = bootstrap::Client::new(&server, link.clone())?;
311 let mut proof = None;
312 if database.seed_publication_intent_bytes().await?.is_none() {
314 progress(Stage::PreparingData.into());
315 let timer = bootstrap::StageTimer::start("prepare_claim");
316 let seed = store
317 .prepare_seed_claim(database, setup_id)
318 .await
319 .map_err(explain_seed_claim_error)?;
320 database.clear_local_seed_claim_refused().await?;
321 drop(timer);
322 let timer = bootstrap::StageTimer::start("claim");
323 let bearer = ClaimAuthentication::SeedBearer(seed.bearer());
324 let claim = match secret {
325 Some(secret) => {
326 let setup = ClaimAuthentication::SetupSecret(secret);
327 match bootstrap.claim(seed.genesis(), setup).await {
328 Ok(()) => Ok(()),
329 Err(_) => bootstrap.claim(seed.genesis(), bearer).await,
332 }
333 }
334 None => bootstrap.claim(seed.genesis(), bearer).await,
335 };
336 drop(timer);
337 if let Err(error) = claim {
338 if setup_refusal(&error) {
341 let fenced = database.seed_source_pin().await?.is_some();
342 if secret.is_none()
345 && (!fenced || has_code(&error, "bootstrap-setup-invitation-rejected"))
346 {
347 return Err(error.context(format!("error {SETUP_INVITATION_REQUIRED}")));
348 }
349 if !fenced {
350 database
351 .mark_local_seed_claim_refused(error.to_string().as_str())
352 .await?;
353 return Err(explain_setup_refusal(error));
354 }
355 return Err(explain_fenced_setup_refusal(error));
356 }
357 store.prepare_seed_source(database).await?;
360 database
361 .capture_local_shared_state_for_setup(&blob_dir)
362 .await?;
363 store
364 .package_seed_capture_validated(database, &blob_dir, setup_id, None)
365 .await?;
366 return Err(error.context(
367 "error sync-setup-outcome-unknown hint=\"the server claim couldn't be confirmed; resume continues the same setup\"",
368 ));
369 }
370 {
371 let _timer = bootstrap::StageTimer::start("prepare_source");
372 store.prepare_seed_source(database).await?;
373 }
374 let capture = {
375 let _timer = bootstrap::StageTimer::start("capture");
376 database
377 .capture_local_shared_state_for_setup(&blob_dir)
378 .await?
379 };
380 let _timer = bootstrap::StageTimer::start("package");
381 proof = Some(
382 store
383 .package_seed_capture_validated(database, &blob_dir, setup_id, Some(capture))
384 .await?,
385 );
386 }
387 progress(Stage::UploadingData.into());
388 bootstrap
389 .resume_validated(&store, database, proof, &|done, total| {
390 progress(Progress {
391 stage: Stage::UploadingData,
392 amount: Some(Amount::Bytes {
393 done,
394 total: Some(total),
395 }),
396 })
397 })
398 .await
399 .map_err(explain_fenced_setup_refusal)?;
400 progress(Stage::FinishingSetup.into());
401 {
403 let _timer = bootstrap::StageTimer::start("enrollment");
404 enrollment::Client::new(&server, link.clone())?
405 .refresh(&store, database)
406 .await?;
407 }
408 let _timer = bootstrap::StageTimer::start("drain");
409 let client = tail::Client::new(&server, link.clone())?;
410 let outcome = drain(&client, &store, database, host, &blob_dir, ROUND_LIMIT).await?;
411 Ok((server, outcome))
412}
413
414pub struct PendingInvitation {
416 server: String,
417 text: Zeroizing<String>,
418 handle: [u8; 32],
419 vault: [u8; 32],
420 deadline: Instant,
421 expires_at: u64,
422 resumed: bool,
423}
424
425#[derive(Clone, Copy, Debug, PartialEq, Eq)]
426pub struct InvitationStatus {
427 pub expires_at: u64,
428 pub keys_may_have_been_sent: bool,
429}
430
431#[derive(Clone, Copy, Debug, PartialEq, Eq)]
432pub enum Cancellation {
433 None,
434 Cancelled,
435 KeysMayHaveBeenSent { expires_at: u64 },
436}
437
438impl PendingInvitation {
439 pub fn text(&self) -> &str {
440 &self.text
441 }
442
443 pub fn expires_at(&self) -> u64 {
444 self.expires_at
445 }
446
447 pub fn resumed(&self) -> bool {
448 self.resumed
449 }
450
451 pub fn server(&self) -> &str {
453 &self.server
454 }
455
456 #[cfg(any(test, feature = "test-support"))]
457 pub fn for_test(server: &str, text: &str, expires_at: u64) -> Self {
458 Self {
459 server: server.to_string(),
460 text: Zeroizing::new(text.to_string()),
461 handle: [0; 32],
462 vault: [0; 32],
463 deadline: Instant::now(),
464 expires_at,
465 resumed: false,
466 }
467 }
468}
469
470impl std::fmt::Debug for PendingInvitation {
471 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
472 f.write_str("PendingInvitation([REDACTED])")
473 }
474}
475
476pub async fn create_invitation(
479 link: Link,
480 database: &Database,
481 host: &dyn ClientHost,
482) -> Result<PendingInvitation> {
483 host.ensure_sync_allowed()?;
484 ensure!(is_set_up(database).await?, NOT_SET_UP);
485 let store = key_store(host, database).await?;
486 let _guard = coordination::acquire(database).await?;
487 let Some((_, server)) = store.association(database).await? else {
488 bail!("error sync-setup-incomplete hint=\"rerun `aven sync setup`\"");
489 };
490 ensure!(
493 server.len() <= super::MAX_SERVER_BYTES,
494 "error sync-server-url-too-long hint=\"invitations need a server origin of at most 255 bytes\""
495 );
496 let now = unix_now()?;
497 let created = enrollment::Client::new(&server, link.clone())?
498 .invite_with_status(&store, database, now + invitation_seconds())
499 .await
500 .map_err(|error| match code(&error).as_deref() {
501 Some("withdrawal-required-unsupported") => error.context(
502 "error sync-invitation-unresolved hint=\"keys may already have been sent with the previous invitation; invite again after that device joins, or after the invitation expires and the next `aven sync` changes keys\"",
503 ),
504 _ => explain_change_limit(error),
505 });
506 let created = track_access_result(database, created).await?;
507 ensure!(
508 created.state.expires_at > now,
509 "error sync-invitation-unused hint=\"the open invitation has expired; run `aven sync` before creating another invitation\""
510 );
511 let text = DeviceInvitation {
512 server: server.clone(),
513 invitation: created.invitation,
514 }
515 .encode()?;
516 Ok(PendingInvitation {
517 server,
518 text,
519 handle: created.state.handle,
520 vault: created.vault,
521 deadline: Instant::now() + Duration::from_secs(created.state.expires_at - now),
522 expires_at: created.state.expires_at,
523 resumed: created.resumed,
524 })
525}
526
527#[derive(Debug, Default)]
530pub struct AssociationStatus {
531 pub server: Option<String>,
532 pub devices: Option<usize>,
533 pub invitation: Option<InvitationStatus>,
534}
535
536pub async fn association_status(
537 database: &Database,
538 host: &dyn ClientHost,
539) -> Result<AssociationStatus> {
540 if !is_set_up(database).await? {
541 return Ok(AssociationStatus::default());
542 }
543 let store = key_store(host, database).await?;
544 let _guard = coordination::acquire(database).await?;
545 read_association(database, &store).await
546}
547
548pub async fn try_association_status(
551 database: &Database,
552 host: &dyn ClientHost,
553) -> Result<Option<AssociationStatus>> {
554 if !is_set_up(database).await? {
555 return Ok(Some(AssociationStatus::default()));
556 }
557 let Some(_guard) = coordination::try_acquire(database)? else {
558 return Ok(None);
559 };
560 let store = key_store(host, database).await?;
561 read_association(database, &store).await.map(Some)
562}
563
564async fn read_association(
565 database: &Database,
566 store: &ProtectedLocalKeyStore,
567) -> Result<AssociationStatus> {
568 let Some((_, server)) = store.association(database).await? else {
569 return Ok(AssociationStatus::default());
570 };
571 let inputs = store.active_inputs(database, &server).await?;
572 let invitation = store
573 .open_invitation(database, &inputs)
574 .await?
575 .map(|state| InvitationStatus {
576 expires_at: state.expires_at,
577 keys_may_have_been_sent: state.keys_may_have_been_sent,
578 });
579 Ok(AssociationStatus {
580 server: Some(server),
581 devices: Some(inputs.membership.device_count()),
582 invitation,
583 })
584}
585
586pub async fn cancel_invitation(
587 link: Link,
588 database: &Database,
589 host: &dyn ClientHost,
590) -> Result<Cancellation> {
591 host.ensure_sync_allowed()?;
592 ensure!(is_set_up(database).await?, NOT_SET_UP);
593 let store = key_store(host, database).await?;
594 let _guard = coordination::acquire(database).await?;
595 let Some((_, server)) = store.association(database).await? else {
596 bail!("error sync-setup-incomplete hint=\"rerun `aven sync setup`\"");
597 };
598 let client = enrollment::Client::new(&server, link.clone())?;
599 let mut inputs = store.active_inputs(database, &server).await?;
600 let (journal, state) = store.retire_open_invitation(database, &inputs).await?;
601 let Some(state) = state else {
602 return Ok(Cancellation::None);
603 };
604 let Some(journal) = journal else {
605 return Ok(Cancellation::KeysMayHaveBeenSent {
606 expires_at: state.expires_at,
607 });
608 };
609 let _ = client
612 .cancel(&store, database, &mut inputs, journal.handle)
613 .await;
614 Ok(Cancellation::Cancelled)
615}
616
617#[derive(Clone, Copy, Debug, PartialEq, Eq)]
619pub enum Admission {
620 Admitted,
621 Cancelled,
623}
624
625const INVITATION_UNUSED: &str = "error sync-invitation-unused hint=\"the invitation expired unused; if keys may have been sent with it, the next `aven sync` changes keys before uploading new changes\"";
626
627enum AdmissionPoll {
628 Waiting,
629 RequestPending,
630 Complete(Admission),
631}
632
633async fn poll_admission(
636 database: &Database,
637 host: &dyn ClientHost,
638 client: &enrollment::Client,
639 invitation: &PendingInvitation,
640) -> Result<AdmissionPoll> {
641 let store = key_store(host, database).await?;
642 let _guard = coordination::acquire(database).await?;
643 match store
644 .invitation_progress(database, &invitation.handle)
645 .await?
646 {
647 InvitationProgress::Admitted => {
648 return Ok(AdmissionPoll::Complete(Admission::Admitted));
649 }
650 InvitationProgress::Closed => {
651 return Ok(AdmissionPoll::Complete(Admission::Cancelled));
652 }
653 InvitationProgress::Open => {}
654 }
655 match client
656 .join_requested(invitation.vault, invitation.handle)
657 .await
658 {
659 Ok(true) => {}
660 Ok(false) => return Ok(AdmissionPoll::Waiting),
661 Err(error) if busy(&error) => return Ok(AdmissionPoll::Waiting),
662 Err(error) => return Err(error),
663 }
664 match client
665 .admit_handle(&store, database, Some(invitation.handle))
666 .await
667 {
668 Err(error) if busy(&error) => Ok(AdmissionPoll::RequestPending),
669 result => Ok(if track_access_result(database, result).await? {
670 AdmissionPoll::Complete(Admission::Admitted)
671 } else {
672 AdmissionPoll::RequestPending
673 }),
674 }
675}
676
677pub async fn await_admission(
680 link: Link,
681 host: &dyn ClientHost,
682 database: &Database,
683 invitation: &PendingInvitation,
684) -> Result<Admission> {
685 let client = enrollment::Client::new(&invitation.server, link.clone())?;
686 let mut pending_attempt = None;
687 while Instant::now() < invitation.deadline {
688 match poll_admission(database, host, &client, invitation).await? {
689 AdmissionPoll::Complete(admission) => return Ok(admission),
690 AdmissionPoll::RequestPending => {
691 pending_attempt.get_or_insert(0);
692 }
693 AdmissionPoll::Waiting => {}
694 }
695 let delay = match pending_attempt.as_mut() {
696 Some(attempt) => {
697 let delay = join_poll_interval(*attempt);
698 *attempt = attempt.saturating_add(1);
699 delay
700 }
701 None => jittered(INVITER_POLL_INTERVAL),
702 };
703 link.wait(delay).await;
704 }
705 bail!(INVITATION_UNUSED)
706}
707
708pub async fn ensure_join_available(database: &Database, host: &dyn ClientHost) -> Result<()> {
711 host.ensure_sync_allowed()?;
712 match local_phase(database).await? {
713 LocalPhase::JoinIncomplete => Ok(()),
714 LocalPhase::NotSetUp => {
715 database
716 .peer_target_preflight()
717 .await
718 .context(JOIN_REQUIRES_EMPTY)?;
719 Ok(())
720 }
721 LocalPhase::SetupIncomplete | LocalPhase::SetUp => {
722 bail!(ALREADY_SET_UP)
723 }
724 }
725}
726
727const JOIN_TIMEOUT: &str = "error sync-join-timeout hint=\"the other device did not add this device in time; keep `aven sync invite` running there and rerun `aven sync join`. If the invitation expired, run `aven sync invite` again on the same device and pass the new invitation to `aven sync join --new-invitation`; an admission from the earlier invitation still completes the join\"";
731
732const ALREADY_SET_UP: &str =
733 "error sync-already-set-up hint=\"add devices with `aven sync invite` on this database\"";
734const JOIN_REQUIRES_EMPTY: &str = "error sync-join-requires-empty-database hint=\"join with a new database, for example `aven --db PATH sync join`\"";
735
736pub async fn run_join(
742 link: Link,
743 database: &Database,
744 host: &dyn ClientHost,
745 invitation: impl FnOnce() -> Result<Option<DeviceInvitation>>,
746 replace: bool,
747 progress: &(dyn Fn(Progress) + Sync),
748) -> Result<(String, Outcome)> {
749 host.ensure_sync_allowed()?;
750 let store = key_store(host, database).await?;
751 let blob_dir = host.blob_dir(database)?;
752 let _guard = coordination::acquire(database).await?;
753 let server = match store.association(database).await? {
754 Some((true, server)) => Some(server),
755 Some((false, _)) => bail!(ALREADY_SET_UP),
756 None => {
757 database
758 .peer_target_preflight()
759 .await
760 .context(JOIN_REQUIRES_EMPTY)?;
761 None
762 }
763 };
764 let server = if matches!(
765 store.enrollment_readiness(database).await?,
766 EnrollmentReadiness::Enrolled { .. }
767 ) {
768 server.context("error sync-join-incomplete")?
769 } else {
770 let invitation = invitation()?;
771 ensure!(
772 server
773 .as_ref()
774 .zip(invitation.as_ref())
775 .is_none_or(|(server, invitation)| *server == invitation.server),
776 "error sync-join-server-mismatch hint=\"this database started joining with another server; use an invitation for that server\""
777 );
778 let (server, invitation) = match invitation {
779 Some(invitation) => (invitation.server, Some(invitation.invitation)),
780 None => (server.context("error sync-join-invitation-required")?, None),
781 };
782 let client = enrollment::Client::new(&server, link.clone())?;
783 let deadline = Instant::now() + Duration::from_secs(invitation_seconds());
784 await_join(
785 &client, &store, database, invitation, replace, deadline, progress,
786 )
787 .await?;
788 server
789 };
790 progress(Stage::DownloadingTasks.into());
791 enrollment::Client::new(&server, link.clone())?
792 .install_reporting(&store, database, &|done, total| {
793 progress(Progress {
794 stage: Stage::DownloadingTasks,
795 amount: Some(Amount::Bytes { done, total }),
796 })
797 })
798 .await?;
799 progress(Stage::CatchingUp.into());
800 let client = tail::Client::new(&server, link.clone())?;
801 let outcome = drain_reporting(
802 &client,
803 &store,
804 database,
805 host,
806 &blob_dir,
807 ROUND_LIMIT,
808 Some(progress),
809 )
810 .await?;
811 Ok((server, outcome))
812}
813
814fn explain_join_refusal(error: anyhow::Error) -> anyhow::Error {
817 let hint = match code(&error).as_deref() {
818 Some("enrollment-invitation-conflict") => {
819 "error sync-join-invitation-conflict hint=\"this database started joining with another invitation; rerun with that invitation to resume, or pass a new invitation from the same inviting device with `aven sync join --new-invitation`\""
820 }
821 Some("enrollment-retry-context") => {
822 "error sync-join-new-invitation-mismatch hint=\"a new invitation must come from the device that created the first one; run `aven sync invite` there\""
823 }
824 Some("enrollment-retry-unavailable") => {
825 "error sync-join-new-invitation-unavailable hint=\"joining already got past admission, so a new invitation cannot be used; rerun `aven sync join` with an invitation this database already used\""
826 }
827 Some("enrollment-retry-limit") => {
828 "error sync-join-new-invitation-limit hint=\"this database has reached its invitation limit; rerun `aven sync join` with an invitation it already used to finish if the other device accepted it, otherwise keep this database unchanged and join from a new empty database, for example `aven --db PATH sync join`\""
829 }
830 Some("shared-state-install") => {
831 "error sync-join-target-not-empty hint=\"data was added to this database while joining, so it cannot finish joining; keep it unchanged and join from a new empty database, for example `aven --db PATH sync join`\""
832 }
833 _ => return error,
834 };
835 error.context(hint)
836}
837
838pub async fn await_join(
843 client: &enrollment::Client,
844 store: &ProtectedLocalKeyStore,
845 database: &Database,
846 invitation: Option<Invitation>,
847 replace: bool,
848 deadline: Instant,
849 progress: &(dyn Fn(Progress) + Sync),
850) -> Result<()> {
851 let peer = client
852 .prepare(store, database, invitation, replace)
853 .await
854 .map_err(explain_join_refusal)?;
855 let mut posted = client.post(&peer).await;
856 progress(Stage::WaitingForInviter.into());
857 let others = client.has_other_attempts(store, database).await?;
858 let mut poll_attempt = 0_usize;
859 loop {
860 if posted.as_ref().is_err_and(busy) {
861 posted = client.post(&peer).await;
862 }
863 match client.complete(store, database).await {
864 Ok(true) => return Ok(()),
865 Err(error) if !busy(&error) => return Err(error),
866 _ => {}
867 }
868 let refused = posted.as_ref().is_err_and(|error| !busy(error));
869 if (refused && !others) || Instant::now() >= deadline {
870 return Err(match posted {
871 Err(error) if refused && !others => error,
872 Err(error) => error.context(JOIN_TIMEOUT),
873 Ok(()) => anyhow::anyhow!(JOIN_TIMEOUT),
874 });
875 }
876 client.link().wait(join_poll_interval(poll_attempt)).await;
877 poll_attempt = poll_attempt.saturating_add(1);
878 }
879}
880
881fn setup_refusal(error: &anyhow::Error) -> bool {
882 matches!(
883 code(error).as_deref(),
884 Some(
885 "bootstrap-storage-already-claimed"
886 | "bootstrap-setup-invitation-rejected"
887 | "bootstrap-setup-invitation-expired"
888 )
889 )
890}
891
892fn explain_fenced_setup_refusal(error: anyhow::Error) -> anyhow::Error {
893 match code(&error).as_deref() {
894 Some("bootstrap-setup-invitation-rejected" | "bootstrap-setup-invitation-expired") => error.context(
895 "error sync-setup-fenced-invitation-rejected hint=\"this setup is already frozen; resume with the invitation that started setup or the newest invitation for that same server storage; if neither is available, back up this database and restore it to a new path for a local-only copy; local editing and export still work\"",
896 ),
897 Some("bootstrap-storage-already-claimed") => error.context(
898 "error sync-setup-fenced-storage-claimed hint=\"the server reported that this storage belongs to another sync; this setup is frozen and resuming retries it; if it keeps failing, back up this database and restore it to a new path for a local-only copy; local editing and export still work\"",
899 ),
900 _ => error,
901 }
902}
903
904fn explain_setup_refusal(error: anyhow::Error) -> anyhow::Error {
905 match code(&error).as_deref() {
906 Some("bootstrap-storage-already-claimed") => error.context(
907 "error sync-setup-storage-already-claimed hint=\"this server already belongs to another sync; nothing here was changed; to use that sync, join it from an empty database\"",
908 ),
909 Some("bootstrap-setup-invitation-rejected") => error.context(
910 "error sync-setup-invitation-rejected hint=\"this setup invitation expired, was replaced, or is for different storage; nothing here was changed; get the current invitation from your hosting provider, or run `aven server setup` if you host the server yourself\"",
911 ),
912 Some("bootstrap-setup-invitation-expired") => error.context(
913 "error sync-setup-invitation-expired hint=\"this setup invitation expired; nothing here was changed; get a new invitation from your hosting provider, or run `aven server setup` again if you host the server yourself\"",
914 ),
915 _ => error,
916 }
917}
918
919fn busy(error: &anyhow::Error) -> bool {
921 has_code(error, "enrollment-busy")
922}
923
924async fn associated_server(store: &ProtectedLocalKeyStore, database: &Database) -> Result<String> {
925 match store.association(database).await? {
926 Some((peer, server)) => {
927 ensure!(
928 !peer
929 || matches!(
930 store.enrollment_readiness(database).await?,
931 EnrollmentReadiness::Enrolled { .. }
932 ),
933 "error sync-join-incomplete hint=\"rerun `aven sync join`\""
934 );
935 Ok(server)
936 }
937 None => bail!("error sync-setup-incomplete hint=\"rerun `aven sync setup`\""),
938 }
939}
940
941const CHANGE_LIMIT: &str = "error sync-device-change-limit hint=\"this sync has reached its limit on device changes, so devices can no longer be added or removed; start a new sync to keep changing devices, see https://aventasks.dev/sync/setup/#recover-from-device-loss\"";
943
944fn explain_change_limit(error: anyhow::Error) -> anyhow::Error {
945 if has_code(&error, "membership-change-limit") {
946 error.context(CHANGE_LIMIT)
947 } else {
948 error
949 }
950}
951
952const REFUSED: &str = "error sync-server-refused hint=\"the server refused this request; it may have failed, or another device may have removed this device from sync, which leaves local tasks and images available here; retry later, and check `aven sync device list` on another device\"";
954
955fn explain_round_error(error: anyhow::Error) -> anyhow::Error {
958 match code(&error).as_deref() {
959 Some("snapshot-not-installed" | "enrollment-unresolved") => {
960 error.context("error sync-join-incomplete hint=\"rerun `aven sync join`\"")
961 }
962 Some("enrollment-unauthorized") => error.context(REFUSED),
963 Some("withdrawal-rotation-required") => error.context(KEY_CHANGE_REQUIRED),
964 _ => error,
965 }
966}
967
968const KEY_CHANGE_REQUIRED: &str = "error sync-key-change-required hint=\"an invitation expired after keys may have been sent to a device that never joined; this device downloads changes but uploads new ones only after sync changes keys; check the connection and run `aven sync` again\"";
970
971async fn remember_access_refusal(database: &Database, error: &anyhow::Error) {
972 if is_access_refusal(error)
973 && let Err(state_error) = database.record_sync_access_refusal().await
974 {
975 tracing::warn!(
976 error = %state_error,
977 "could not persist sync access refusal"
978 );
979 }
980}
981
982async fn track_access_result<T>(database: &Database, result: Result<T>) -> Result<T> {
984 match result {
985 Ok(value) => {
986 database.clear_sync_access_refusal().await?;
987 Ok(value)
988 }
989 Err(error) => {
990 remember_access_refusal(database, &error).await;
991 Err(error)
992 }
993 }
994}
995
996async fn drain_associated(
999 link: Link,
1000 database: &Database,
1001 host: &dyn ClientHost,
1002 round_limit: usize,
1003) -> Result<Outcome> {
1004 let store = key_store(host, database).await?;
1005 let blob_dir = host.blob_dir(database)?;
1006 let server = associated_server(&store, database).await?;
1007 let client = tail::Client::new(&server, link.clone())?;
1008 let result = drain(&client, &store, database, host, &blob_dir, round_limit)
1009 .await
1010 .map_err(explain_round_error);
1011 match result {
1012 Ok(outcome) => {
1013 database.clear_sync_access_refusal().await?;
1014 Ok(outcome)
1015 }
1016 Err(error) => {
1017 remember_access_refusal(database, &error).await;
1018 Err(error)
1019 }
1020 }
1021}
1022
1023pub async fn run_to_completion(
1025 link: Link,
1026 database: &Database,
1027 host: &dyn ClientHost,
1028) -> Result<Outcome> {
1029 host.ensure_sync_allowed()?;
1030 ensure!(is_set_up(database).await?, NOT_SET_UP);
1031 let _guard = coordination::acquire(database).await?;
1032 drain_associated(link, database, host, ROUND_LIMIT).await
1033}
1034
1035pub enum DaemonRound {
1036 Completed(Outcome),
1037 Deferred,
1039 NotSetUp,
1041}
1042
1043pub async fn daemon_round(
1045 link: Link,
1046 database: &Database,
1047 host: &dyn ClientHost,
1048 round_limit: usize,
1049) -> Result<DaemonRound> {
1050 host.ensure_sync_allowed()?;
1051 if !is_set_up(database).await? {
1052 return Ok(DaemonRound::NotSetUp);
1053 }
1054 let Some(_guard) = coordination::try_acquire(database)? else {
1055 return Ok(DaemonRound::Deferred);
1056 };
1057 drain_associated(link, database, host, round_limit)
1058 .await
1059 .map(DaemonRound::Completed)
1060}
1061
1062#[derive(Debug, Serialize)]
1063pub struct Outcome {
1064 pub version: u32,
1065 pub rounds: usize,
1066 pub metadata_caught_up: bool,
1067 pub images: ImageTransfer,
1068 pub sent_changes: usize,
1069 pub received_changes: usize,
1070 pub conflicts: usize,
1071 pub new_conflicts: usize,
1072}
1073
1074impl Outcome {
1075 pub fn more_work_ready(&self) -> bool {
1078 self.images == ImageTransfer::Pending
1079 || (!self.metadata_caught_up && self.images == ImageTransfer::Complete)
1080 }
1081}
1082
1083pub async fn drain(
1086 client: &tail::Client,
1087 store: &ProtectedLocalKeyStore,
1088 database: &Database,
1089 host: &dyn ClientHost,
1090 blob_dir: &Path,
1091 round_limit: usize,
1092) -> Result<Outcome> {
1093 drain_reporting(client, store, database, host, blob_dir, round_limit, None).await
1094}
1095
1096async fn drain_reporting(
1097 client: &tail::Client,
1098 store: &ProtectedLocalKeyStore,
1099 database: &Database,
1100 host: &dyn ClientHost,
1101 blob_dir: &Path,
1102 round_limit: usize,
1103 progress: Option<&(dyn Fn(Progress) + Sync)>,
1104) -> Result<Outcome> {
1105 let conflicts_before = conflict_identities(database).await?;
1106 super::device_label::publish_if_missing(database, store, host).await?;
1107 let mut last = None;
1108 let mut rounds = 0;
1109 let mut image_retries = 0;
1110 let mut sent_changes = 0;
1111 let mut received_changes = 0;
1112 let mut images_progress = None;
1113 let lifecycle_policy = host.attachment_lifecycle_policy();
1114 let mut drain = Box::pin(client.start_drain(store, database)).await?;
1115 while rounds < round_limit {
1116 let round: Round = Box::pin(client.round_in_drain(
1117 store,
1118 database,
1119 blob_dir,
1120 lifecycle_policy,
1121 &mut drain,
1122 ))
1123 .await?;
1124 rounds += 1;
1125 sent_changes += round.sent_changes;
1126 received_changes += round.received_changes;
1127 if let Some(progress) = progress {
1128 images_progress = report_round(
1129 database,
1130 &round,
1131 received_changes,
1132 images_progress,
1133 progress,
1134 )
1135 .await?;
1136 }
1137 last = Some(round);
1138 match round.images {
1139 ImageTransfer::Complete if round.metadata_caught_up => break,
1140 ImageTransfer::Complete | ImageTransfer::Pending => image_retries = 0,
1141 ImageTransfer::Failed | ImageTransfer::Unavailable => {
1142 image_retries += 1;
1143 if image_retries >= IMAGE_RETRY_ROUNDS {
1144 break;
1145 }
1146 }
1147 }
1148 }
1149 let last = last.context("error sync-no-rounds")?;
1150 if last.publishing_blocked {
1151 return Err(drain.publishing_blocked_error());
1152 }
1153 let conflicts_after = conflict_identities(database).await?;
1154 Ok(Outcome {
1155 version: 1,
1156 rounds,
1157 metadata_caught_up: last.metadata_caught_up,
1158 images: last.images,
1159 sent_changes,
1160 received_changes,
1161 conflicts: conflicts_after.len(),
1162 new_conflicts: conflicts_after.difference(&conflicts_before).count(),
1163 })
1164}
1165
1166async fn report_round(
1171 database: &Database,
1172 round: &Round,
1173 received_changes: usize,
1174 images: Option<(u64, u64)>,
1175 progress: &(dyn Fn(Progress) + Sync),
1176) -> Result<Option<(u64, u64)>> {
1177 if !round.metadata_caught_up {
1178 progress(Progress {
1179 stage: Stage::CatchingUp,
1180 amount: Some(Amount::Changes {
1181 done: received_changes as u64,
1182 }),
1183 });
1184 return Ok(images);
1185 }
1186 if round.images == ImageTransfer::Complete {
1187 return Ok(images);
1188 }
1189 let remaining = database.encrypted_image_downloads_remaining().await?;
1190 let done = images_done(images, remaining);
1191 progress(Progress {
1192 stage: Stage::DownloadingImages,
1193 amount: Some(Amount::Images { done, remaining }),
1194 });
1195 Ok(Some((done, remaining)))
1196}
1197
1198fn images_done(previous: Option<(u64, u64)>, remaining: u64) -> u64 {
1200 previous.map_or(0, |(done, before)| done + before.saturating_sub(remaining))
1201}
1202
1203async fn conflict_identities(database: &Database) -> Result<HashSet<String>> {
1204 let mut identities = HashSet::new();
1205 for workspace in database.list_workspaces().await? {
1206 for conflict in database.list_conflicts(&workspace, None, None).await? {
1207 let (first, second) = if conflict.variant_a <= conflict.variant_b {
1208 (conflict.variant_a, conflict.variant_b)
1209 } else {
1210 (conflict.variant_b, conflict.variant_a)
1211 };
1212 identities.insert(format!(
1213 "{}:{}:{}:{}:{}:{}",
1214 workspace.id,
1215 conflict.recurrence_series,
1216 conflict.task_id,
1217 conflict.field,
1218 first,
1219 second
1220 ));
1221 }
1222 }
1223 Ok(identities)
1224}
1225
1226#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
1227#[serde(rename_all = "kebab-case")]
1228pub enum SyncState {
1229 NotSetUp,
1230 SetupIncomplete,
1231 JoinIncomplete,
1232 KeyChangePending,
1233 Ready,
1234 AccessRefused,
1235}
1236
1237#[derive(Serialize)]
1238pub struct StatusReport {
1239 pub version: u32,
1240 pub devices: Option<usize>,
1242 pub server: Option<String>,
1243 pub state: SyncState,
1244 pub local_changes_pending: Option<bool>,
1245 pub server_position: Option<i64>,
1246 pub initial_download_pending: Option<bool>,
1247 pub image_uploads_pending: Option<bool>,
1248 pub image_downloads_pending: Option<bool>,
1249 pub images_unavailable: Option<bool>,
1250 pub conflicts: i64,
1251 pub invitation: &'static str,
1252 pub invitation_expires_at: Option<u64>,
1253 pub access_refused_at: Option<String>,
1254}
1255
1256pub async fn status_report(database: &Database, host: &dyn ClientHost) -> Result<StatusReport> {
1259 let mut report = StatusReport {
1260 version: 1,
1261 devices: None,
1262 server: None,
1263 state: SyncState::NotSetUp,
1264 local_changes_pending: None,
1265 server_position: None,
1266 initial_download_pending: None,
1267 image_uploads_pending: None,
1268 image_downloads_pending: None,
1269 images_unavailable: None,
1270 conflicts: database.unresolved_conflict_count().await?,
1271 invitation: "none",
1272 invitation_expires_at: None,
1273 access_refused_at: database
1274 .sync_access_refusal()
1275 .await?
1276 .map(|refusal| refusal.at),
1277 };
1278 if !is_set_up(database).await? {
1279 return Ok(report);
1280 }
1281 report.state = SyncState::SetupIncomplete;
1282 let store = key_store(host, database).await?;
1283 let _guard = coordination::acquire(database).await?;
1284 if let Some((peer, server)) = store.association(database).await? {
1285 report.server = Some(server.clone());
1286 let enrollment_ready = !peer
1287 || matches!(
1288 store.enrollment_readiness(database).await?,
1289 EnrollmentReadiness::Enrolled { .. }
1290 );
1291 if enrollment_ready {
1292 let invitation_inputs = store.active_inputs(database, &server).await?;
1293 report.devices = Some(invitation_inputs.membership.device_count());
1294 if let Some(invitation) = store.open_invitation(database, &invitation_inputs).await? {
1295 report.invitation = "open";
1296 report.invitation_expires_at = Some(invitation.expires_at);
1297 }
1298 }
1299 report.state = if !enrollment_ready {
1300 SyncState::JoinIncomplete
1301 } else {
1302 match store.tail_inputs(database, &server).await {
1303 Ok(inputs) => {
1304 let state = database.encrypted_round_state(&inputs.authority).await?;
1305 report.local_changes_pending = Some(!state.idle);
1306 report.server_position = Some(state.cursor);
1307 report.initial_download_pending = Some(state.downloads.is_none());
1308 report.image_uploads_pending = Some(state.upload_pending);
1309 report.image_downloads_pending = state.downloads.map(|d| d.pending);
1310 report.images_unavailable = state.downloads.map(|d| d.unavailable);
1311 if inputs.publishing_blocked()? {
1312 SyncState::KeyChangePending
1313 } else {
1314 SyncState::Ready
1315 }
1316 }
1317 Err(error) if peer && has_code(&error, "snapshot-not-installed") => {
1318 SyncState::JoinIncomplete
1319 }
1320 Err(error) => return Err(error),
1321 }
1322 };
1323 }
1324 if report.access_refused_at.is_some()
1325 && !matches!(
1326 report.state,
1327 SyncState::SetupIncomplete | SyncState::JoinIncomplete
1328 )
1329 {
1330 report.state = SyncState::AccessRefused;
1331 }
1332 Ok(report)
1333}
1334
1335#[cfg(test)]
1336mod tests {
1337 use super::*;
1338 use crate::sync::client::errors::has_code;
1339
1340 #[test]
1341 fn enrollment_polling_backs_off_to_a_bounded_steady_interval() {
1342 assert_eq!(join_poll_interval(0), Duration::from_millis(25));
1343 assert_eq!(join_poll_interval(1), Duration::from_millis(50));
1344 assert_eq!(join_poll_interval(2), Duration::from_millis(75));
1345 assert_eq!(join_poll_interval(3), Duration::from_millis(100));
1346 assert_eq!(join_poll_interval(usize::MAX), Duration::from_millis(100));
1347 assert_eq!(jittered(INVITER_POLL_INTERVAL), Duration::from_millis(100));
1348 }
1349
1350 #[test]
1351 fn images_done_counts_only_drops_in_remaining_images() {
1352 assert_eq!(images_done(None, 10), 0);
1353 assert_eq!(images_done(Some((0, 10)), 9), 1);
1354 assert_eq!(images_done(Some((1, 9)), 9), 1);
1356 assert_eq!(images_done(Some((1, 9)), 12), 1);
1358 assert_eq!(images_done(Some((1, 12)), 11), 2);
1359 }
1360
1361 #[test]
1362 fn seed_claim_only_labels_a_real_setup_mismatch_as_invitation_mismatch() {
1363 let mismatch = anyhow::Error::new(ProtectedLocalKeyStoreError::new(
1364 ProtectedLocalKeyStoreErrorKind::SetupMismatch,
1365 ));
1366 let mismatch = explain_seed_claim_error(mismatch);
1367 assert!(
1368 has_code(&mismatch, "sync-setup-invitation-mismatch"),
1369 "{mismatch:#}"
1370 );
1371 for kind in [
1372 ProtectedLocalKeyStoreErrorKind::Unavailable,
1373 ProtectedLocalKeyStoreErrorKind::Corrupt,
1374 ] {
1375 let error = explain_seed_claim_error(anyhow::Error::new(
1376 ProtectedLocalKeyStoreError::new(kind),
1377 ));
1378 assert!(
1379 !has_code(&error, "sync-setup-invitation-mismatch"),
1380 "{error:#}"
1381 );
1382 }
1383 }
1384
1385 #[test]
1386 fn join_timeout_hint_covers_expiry_without_suggesting_discarding_data() {
1387 let hint = JOIN_TIMEOUT;
1388 assert!(hint.starts_with("error sync-join-timeout hint="));
1389 assert!(hint.contains("rerun `aven sync join`"));
1390 assert!(hint.contains("`aven sync join --new-invitation`"));
1391 for word in ["delete", "reset", "disposable", "is empty"] {
1392 assert!(!hint.contains(word), "{hint}");
1393 }
1394 }
1395
1396 #[test]
1397 fn device_change_limit_hint_points_to_starting_a_new_sync() {
1398 let error = explain_change_limit(anyhow::anyhow!("error membership-change-limit"));
1399 let hint = error.to_string();
1400 assert!(
1401 hint.starts_with("error sync-device-change-limit hint="),
1402 "{hint}"
1403 );
1404 assert!(hint.contains("#recover-from-device-loss"), "{hint}");
1405 let other = explain_change_limit(anyhow::anyhow!("error membership-invalid"));
1406 assert_eq!(other.to_string(), "error membership-invalid");
1407 }
1408}