use std::cell::Cell;
use std::collections::hash_map::DefaultHasher;
use std::collections::{BTreeSet, VecDeque};
use std::ffi::OsStr;
use std::fmt;
use std::hash::{Hash, Hasher};
use std::io::{self, BufReader, Read, Seek, SeekFrom, Write};
use std::path::{Component, Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, PoisonError, TryLockError};
use std::time::{Duration, Instant};
use serde::de::{DeserializeSeed, Deserializer, IgnoredAny, MapAccess, Visitor};
use serde::{Deserialize, Serialize};
use super::history::{scan_keys, HistoryLocalStore, LOCAL_WINDOW_ROWS};
use super::ledger_guard::{scope_of_head, Disk, LedgerGuard, ScopeState};
use super::outbox::{Outbox, MAX_OUTBOX_ENTRIES};
use super::remote_window::{RemoteWindow, WindowedBackend};
use super::scheduler::{Scheduler, SchedulerPolicy, PULL, PUSH};
use super::traits::{NativeObserver, Spawner, SyncTransport, TranscriptHost};
use crate::transcripts::{
self as sync_core, ConsentScopeBinding, LegacyFiles, LocalVersions, StateFileNames,
TranscriptMapper, TranscriptSyncProfile,
};
use crate::{
AuthProvider, Backend, FileStorage, LedgerStorage, PassOutcome, PassReport, SyncConfig,
SyncEngine, SyncEngineBuilder, SyncError, SyncObserver, SyncResult,
};
#[cfg(test)]
mod recovery_commit_tests {
use super::super::traits::HistorySession;
use super::*;
use crate::recovery_marker::{RecoveryConsent, ReplayReceipt};
static NEXT: AtomicU64 = AtomicU64::new(0);
struct Host(PathBuf);
impl TranscriptHost for Host {
fn data_file(&self, name: &str) -> PathBuf {
self.0.join(name)
}
fn open_history(&self) -> Result<Box<dyn HistorySession>, String> {
Err("history is outside this consent-only test".to_string())
}
fn history_changed(&self) {}
fn busy_reason(&self) -> Option<&'static str> {
None
}
fn recovery_pending(&self, _scope: &str) -> bool {
false
}
fn recovery_complete(&self, _scope: &str, _generation: u64) -> Result<(), String> {
Ok(())
}
fn now_ms(&self) -> u64 {
0
}
fn downloaded(&self, _count: u32) {}
}
struct Files {
root: PathBuf,
consent: ConsentStore,
scheduler: Scheduler,
}
impl Files {
fn new() -> Self {
let root = std::env::temp_dir().join(format!(
"rightkit-recovery-consent-{}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir(&root).unwrap();
let consent = ConsentStore {
host: Arc::new(Host(root.clone())),
file: "consent.json".to_string(),
fence_file: "fence.json".to_string(),
revocation_incomplete: AtomicBool::new(false),
stop_unverified: AtomicBool::new(false),
disable_requests: AtomicU64::new(0),
stop_gate: Mutex::new(()),
seen_epoch: AtomicU64::new(0),
outbox_file: root.join("outbox.json"),
outbox_lock: Arc::new(Mutex::new(())),
account_changed_message: "account changed".to_string(),
lock: Mutex::new(()),
};
{
let _guard = consent.transition_guard().unwrap();
let state = TranscriptSyncState {
sync_enabled: true,
epoch: 41,
scope: ConsentScopeBinding {
authorized_scope: Some("owner".to_string()),
..Default::default()
},
..Default::default()
};
consent.write_locked(&state).unwrap();
consent.write_fence(FenceState::Enabled, 41).unwrap();
}
Self {
root,
consent,
scheduler: Scheduler::new(SchedulerPolicy::new(5, 5, 1), |_| {}),
}
}
fn request_stop(&self) {
let _gate = self.consent.stop_gate.lock().unwrap();
self.consent.disable_requests.fetch_add(1, Ordering::AcqRel);
self.consent
.revocation_incomplete
.store(true, Ordering::Release);
self.scheduler.disable();
}
fn durable_revoke(&self) {
let _guard = self.consent.transition_guard().unwrap();
let mut state = self.consent.read_checked().unwrap();
state.sync_enabled = false;
self.consent
.commit_transition(&mut state, true, true, 0)
.unwrap();
}
}
impl Drop for Files {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
fn proof() -> (ReplayReceipt, RecoveryConsent) {
(
ReplayReceipt {
scope: "owner".to_string(),
consent_generation: 0,
cursor_committed: true,
},
RecoveryConsent {
scope: "owner".to_string(),
epoch: 41,
},
)
}
#[test]
fn revocation_before_admission_never_runs_marker_commit() {
let files = Files::new();
let (receipt, consent) = proof();
files.request_stop();
let mut called = false;
let admitted = files
.consent
.commit_recovery_if_authorized(&files.scheduler, &receipt, &consent, &mut || {
called = true;
Ok(())
})
.unwrap();
assert!(!admitted);
assert!(!called);
files.durable_revoke();
assert!(!files.consent.read_checked().unwrap().sync_enabled);
}
#[test]
fn shutdown_before_admission_refuses_even_with_trusted_durable_consent() {
let files = Files::new();
let (receipt, consent) = proof();
{
let _gate = files.consent.stop_gate.lock().unwrap();
files.scheduler.stop();
}
assert!(files.consent.recovery_state().unwrap().is_some());
let mut called = false;
let admitted = files
.consent
.commit_recovery_if_authorized(&files.scheduler, &receipt, &consent, &mut || {
called = true;
Ok(())
})
.unwrap();
assert!(!admitted);
assert!(!called);
}
#[test]
fn admitted_marker_commit_precedes_durable_revoke_without_blocking_stop_gate() {
let files = Files::new();
let (receipt, consent) = proof();
let (entered_tx, entered_rx) = std::sync::mpsc::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel();
std::thread::scope(|threads| {
let files = &files;
let committing = threads.spawn(move || {
files
.consent
.commit_recovery_if_authorized(
&files.scheduler,
&receipt,
&consent,
&mut || {
entered_tx.send(()).unwrap();
release_rx.recv_timeout(Duration::from_secs(5)).unwrap();
assert!(matches!(
files.consent.load_fence().unwrap(),
FenceRead::Valid(FenceState::Enabled, 41)
));
replace_file_atomically(&files.root.join("marker.json"), b"retained")
.map_err(|error| error.to_string())
},
)
.unwrap()
});
entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
assert!(files.consent.stop_gate.try_lock().is_ok());
files.request_stop();
assert_ne!(files.scheduler.consent_generation(), 0);
assert!(matches!(
files.consent.lock.try_lock(),
Err(TryLockError::WouldBlock)
));
release_tx.send(()).unwrap();
assert!(committing.join().unwrap());
});
assert_eq!(
std::fs::read(files.root.join("marker.json")).unwrap(),
b"retained"
);
files.durable_revoke();
assert!(matches!(
files.consent.load_fence().unwrap(),
FenceRead::Valid(FenceState::Revoked, 42)
));
}
}
const MAX_CONSENT_FILE_BYTES: u64 = 1024 * 1024;
const MAX_FENCE_FILE_BYTES: u64 = 4096;
pub(super) const MAX_LEDGER_FILE_BYTES: u64 = 384 * 1024 * 1024;
const MAX_LEDGER_ENTRIES: usize = 10_600_000;
const MAX_LEGACY_ACKS_FILE_BYTES: u64 = 64 * 1024 * 1024;
const MAX_HISTORY_ROWS_LIMIT: usize = 1_000_000;
const MAX_VERSIONS_FILE_BYTES: u64 = 256 * 1024 * 1024;
const MAX_VERSIONS_ENTRIES: usize = 1_100_000;
const MAX_DRAIN_PER_PASS: usize = 20_000;
const MAX_UPLOADS_PER_PASS: usize = 5_000;
const STATUS_CHECK_TIMEOUT: Duration = Duration::from_secs(15);
const FEED_MAX_IMMEDIATE_PARTIAL_PASSES: u32 = 16;
const FEED_RECENT_CURSORS: usize = 64;
const FEED_MAX_CUT_PASSES_WITHOUT_PROGRESS: u32 = 4;
const FEED_BACKOFF_BASE_SECS: u64 = 60;
const FEED_BACKOFF_MAX_SECS: u64 = 3_600;
#[derive(Default)]
struct FeedWatch {
recent: VecDeque<u64>,
cut_streak: u32,
halt_streak: u32,
immediate_streak: u32,
}
struct FeedFacts<'a> {
stalled: bool,
cut: bool,
end_token: Option<&'a [u8]>,
applied: bool,
}
enum FeedVerdict {
Continue,
Halt(&'static str),
}
fn cursor_digest(token: &[u8]) -> u64 {
let mut hasher = DefaultHasher::new();
token.hash(&mut hasher);
hasher.finish()
}
impl FeedWatch {
fn judge(&mut self, facts: &FeedFacts<'_>) -> FeedVerdict {
if facts.stalled {
self.halt_streak = self.halt_streak.saturating_add(1);
return FeedVerdict::Halt("stalled");
}
if !facts.cut {
*self = Self {
immediate_streak: self.immediate_streak,
..Self::default()
};
return FeedVerdict::Continue;
}
let digest = facts.end_token.map(cursor_digest);
let fresh = match digest {
Some(digest) if !self.recent.contains(&digest) => Some(digest),
_ => None,
};
let Some(digest) = fresh else {
self.halt_streak = self.halt_streak.saturating_add(1);
return FeedVerdict::Halt("cycle");
};
if self.recent.len() >= FEED_RECENT_CURSORS {
self.recent.pop_front();
}
self.recent.push_back(digest);
if facts.applied {
self.cut_streak = 0;
self.halt_streak = 0;
return FeedVerdict::Continue;
}
self.cut_streak = self.cut_streak.saturating_add(1);
if self.cut_streak >= FEED_MAX_CUT_PASSES_WITHOUT_PROGRESS {
self.halt_streak = self.halt_streak.saturating_add(1);
return FeedVerdict::Halt("no_progress");
}
FeedVerdict::Continue
}
fn note_immediate(&mut self) -> bool {
self.immediate_streak = self.immediate_streak.saturating_add(1);
if self.immediate_streak >= FEED_MAX_IMMEDIATE_PARTIAL_PASSES {
self.immediate_streak = 0;
true
} else {
false
}
}
fn backoff(&self) -> Duration {
let exponent = self.halt_streak.saturating_sub(1).min(6);
let factor = 1u64.checked_shl(exponent).unwrap_or(64);
let secs = FEED_BACKOFF_BASE_SECS
.checked_mul(factor)
.unwrap_or(FEED_BACKOFF_MAX_SECS)
.clamp(FEED_BACKOFF_BASE_SECS, FEED_BACKOFF_MAX_SECS);
Duration::from_secs(secs)
}
}
#[derive(Debug, Clone)]
struct DeleteToken {
scope: String,
generation: u64,
epoch: u64,
}
const PASS_TOKEN_MISMATCH: &str = "delete_everywhere_token_mismatch";
pub struct TranscriptSyncConfig {
pub profile: TranscriptSyncProfile,
pub consent_file: String,
pub local_source: String,
pub account_changed_message: String,
pub scope_unavailable_message: String,
pub max_history_rows: usize,
pub scheduler: SchedulerPolicy,
pub event_sink: fn(&str, &str),
pub spawn: Spawner,
}
fn check_text(what: &str, value: &str) -> Result<(), String> {
if value.is_empty() || value.len() > 512 || value.chars().any(char::is_control) {
return Err(format!(
"sync config: {what} is empty, longer than 512 bytes or has control characters"
));
}
Ok(())
}
fn check_file_name(what: &str, value: &str) -> Result<(), String> {
check_text(what, value)?;
let mut parts = Path::new(value).components();
match (parts.next(), parts.next()) {
(Some(Component::Normal(name)), None)
if name == OsStr::new(value) && value.len() <= 128 =>
{
Ok(())
}
_ => Err(format!("sync config: {what} must be one plain file name")),
}
}
impl TranscriptSyncConfig {
fn validate(&self) -> Result<(), String> {
let p = &self.profile;
for (what, value) in [
("record_type", p.record_type.as_str()),
("id_domain", p.id_domain.as_str()),
("scope_domain", p.scope_domain.as_str()),
("default_source", p.default_source.as_str()),
(
"helper_bundle_production",
p.helper_bundle_production.as_str(),
),
(
"helper_bundle_development",
p.helper_bundle_development.as_str(),
),
("local_source", self.local_source.as_str()),
(
"account_changed_message",
self.account_changed_message.as_str(),
),
(
"scope_unavailable_message",
self.scope_unavailable_message.as_str(),
),
] {
check_text(what, value)?;
}
for (what, value) in [
("files.ledger", p.files.ledger.as_str()),
("files.versions", p.files.versions.as_str()),
("files.outbox", p.files.outbox.as_str()),
(
"files.legacy_tombstones",
p.files.legacy_tombstones.as_str(),
),
("files.legacy_pending", p.files.legacy_pending.as_str()),
("files.legacy_aliases", p.files.legacy_aliases.as_str()),
("files.legacy_acks", p.files.legacy_acks.as_str()),
("consent_file", self.consent_file.as_str()),
] {
check_file_name(what, value)?;
}
if p.files.ledger.len() > 100 {
return Err(
"sync config: files.ledger must be at most 100 bytes (the ledger lock file name adds a suffix)"
.to_string(),
);
}
if !(1..=MAX_HISTORY_ROWS_LIMIT).contains(&self.max_history_rows) {
return Err(format!(
"sync config: max_history_rows must be 1..={MAX_HISTORY_ROWS_LIMIT}"
));
}
Ok(())
}
}
pub(super) fn too_large() -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, "state file larger than its cap")
}
pub(super) fn open_state_file(path: &Path) -> io::Result<std::fs::File> {
rightkit_fs::open_read_nofollow(path, &rightkit_fs::ReadOptions::new()).map_err(|error| {
match error {
rightkit_fs::FsError::Io { source, .. } => source,
_ => io::Error::new(
io::ErrorKind::InvalidData,
"state file is a link or not a regular file",
),
}
})
}
pub(super) fn read_capped(path: &Path, cap: u64) -> io::Result<Vec<u8>> {
let file = open_state_file(path)?;
if file.metadata()?.len() > cap {
return Err(too_large());
}
let mut bytes = Vec::new();
file.take(cap.saturating_add(1)).read_to_end(&mut bytes)?;
if bytes.len() as u64 > cap {
return Err(too_large());
}
Ok(bytes)
}
fn stream_json<T: serde::de::DeserializeOwned>(path: &Path, cap: u64) -> io::Result<Option<T>> {
let file = match open_state_file(path) {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error),
};
if file.metadata()?.len() > cap {
return Err(too_large());
}
let mut limited = file.take(cap.saturating_add(1));
let parsed = serde_json::from_reader::<_, T>(BufReader::new(&mut limited));
if limited.limit() == 0 {
return Err(too_large());
}
parsed.map(Some).map_err(|error| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("state file does not decode ({:?})", error.classify()),
)
})
}
pub(super) fn replace_file_atomically(path: &Path, bytes: &[u8]) -> io::Result<()> {
let options = rightkit_fs::atomic::AtomicWriteOptions {
create_parents: true,
..Default::default()
};
rightkit_fs::atomic::atomic_write_with_options(path, options, |file| file.write_all(bytes))
.map_err(|error| match error {
rightkit_fs::FsError::Io { source, .. }
| rightkit_fs::FsError::ReplacedButDirSyncFailed { source, .. } => source,
_ => io::Error::other("atomic write refused"),
})
}
fn read_scope(path: &Path, cap: u64) -> io::Result<Option<String>> {
#[derive(Deserialize)]
struct ScopeOnly {
#[serde(default)]
scope: Option<String>,
}
let mut file = match open_state_file(path) {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error),
};
if file.metadata()?.len() > cap {
return Err(too_large());
}
let mut head = Vec::with_capacity(4096);
(&mut file).take(4096).read_to_end(&mut head)?;
match scope_of_head(&head) {
ScopeState::Bound(scope) => return Ok(Some(scope)),
ScopeState::Unbound => return Ok(None),
ScopeState::Unknown => {}
}
file.seek(SeekFrom::Start(0))?;
let mut limited = file.take(cap.saturating_add(1));
let parsed = serde_json::from_reader::<_, ScopeOnly>(BufReader::new(&mut limited));
if limited.limit() == 0 {
return Err(too_large());
}
parsed.map(|state| state.scope).map_err(|error| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("state file does not decode ({:?})", error.classify()),
)
})
}
#[derive(Default)]
struct KeySet(Vec<String>);
impl<'de> Deserialize<'de> for KeySet {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct KeysVisitor;
impl<'de> Visitor<'de> for KeysVisitor {
type Value = KeySet;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a JSON object")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<KeySet, A::Error> {
let mut keys = Vec::new();
while let Some(key) = map.next_key::<String>()? {
if keys.len() >= MAX_OUTBOX_ENTRIES {
return Err(<A::Error as serde::de::Error>::custom(
"more ids than the cap",
));
}
map.next_value::<IgnoredAny>()?;
keys.push(key);
}
Ok(KeySet(keys))
}
}
deserializer.deserialize_map(KeysVisitor)
}
}
struct LedgerIds {
scope: Option<String>,
remote_ids: Vec<String>,
}
fn read_ledger_ids(path: &Path) -> io::Result<LedgerIds> {
#[derive(Deserialize)]
struct Shape {
#[serde(default)]
scope: Option<String>,
#[serde(default)]
remote_versions: KeySet,
}
Ok(match stream_json::<Shape>(path, MAX_LEDGER_FILE_BYTES)? {
Some(shape) => LedgerIds {
scope: shape.scope,
remote_ids: shape.remote_versions.0,
},
None => LedgerIds {
scope: None,
remote_ids: Vec::new(),
},
})
}
struct Tally {
remaining: Cell<usize>,
over: Cell<bool>,
}
impl Tally {
fn take<E: serde::de::Error>(&self) -> Result<(), E> {
match self.remaining.get().checked_sub(1) {
Some(left) => {
self.remaining.set(left);
Ok(())
}
None => {
self.over.set(true);
Err(E::custom("ledger holds more entries than the cap"))
}
}
}
}
struct CountSeed<'a>(&'a Tally);
impl<'de> DeserializeSeed<'de> for CountSeed<'_> {
type Value = ();
fn deserialize<D: Deserializer<'de>>(self, deserializer: D) -> Result<(), D::Error> {
deserializer.deserialize_any(CountVisitor(self.0))
}
}
struct CountVisitor<'a>(&'a Tally);
impl<'de> Visitor<'de> for CountVisitor<'_> {
type Value = ();
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a map or a list")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<(), A::Error> {
while map.next_key::<IgnoredAny>()?.is_some() {
map.next_value::<IgnoredAny>()?;
self.0.take::<A::Error>()?;
}
Ok(())
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<(), A::Error> {
while seq.next_element::<IgnoredAny>()?.is_some() {
self.0.take::<A::Error>()?;
}
Ok(())
}
fn visit_unit<E: serde::de::Error>(self) -> Result<(), E> {
Ok(())
}
}
struct ShapeVisitor<'a>(&'a Tally);
impl<'de> Visitor<'de> for ShapeVisitor<'_> {
type Value = ();
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a ledger")
}
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<(), A::Error> {
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"tombstones" | "pending_tombstones" | "remote_versions" | "aliases"
| "pending_uploads" | "pending_apply" | "flags" | "withheld" | "extras" => {
map.next_value_seed(CountSeed(self.0))?;
}
_ => {
map.next_value::<IgnoredAny>()?;
}
}
}
Ok(())
}
}
fn check_ledger_entries(bytes: &[u8]) -> Result<(), String> {
let tally = Tally {
remaining: Cell::new(MAX_LEDGER_ENTRIES),
over: Cell::new(false),
};
let mut deserializer = serde_json::Deserializer::from_slice(bytes);
let _ = (&mut deserializer).deserialize_map(ShapeVisitor(&tally));
if tally.over.get() {
Err("cloud sync ledger holds more entries than the cap".to_string())
} else {
Ok(())
}
}
struct BoundedLedgerStorage {
path: PathBuf,
writer: FileStorage,
guard: Arc<LedgerGuard>,
}
impl BoundedLedgerStorage {
fn new(path: PathBuf, guard: Arc<LedgerGuard>) -> Self {
Self {
writer: FileStorage::new(path.clone()),
path,
guard,
}
}
}
impl LedgerStorage for BoundedLedgerStorage {
fn load(&self) -> Result<Option<Vec<u8>>, String> {
let bytes = match read_capped(&self.path, MAX_LEDGER_FILE_BYTES) {
Ok(bytes) => bytes,
Err(error) if error.kind() == io::ErrorKind::NotFound => {
self.guard.record_loaded(None);
return Ok(None);
}
Err(error) => return Err(format!("read cloud sync ledger: {error}")),
};
check_ledger_entries(&bytes)?;
self.guard.record_loaded(Some(&bytes));
Ok(Some(bytes))
}
fn save(&self, bytes: &[u8]) -> Result<(), String> {
if bytes.len() as u64 > MAX_LEDGER_FILE_BYTES {
return Err("cloud sync ledger is larger than its cap".to_string());
}
self.guard
.replace(bytes, |bytes| self.writer.save_locked(bytes))
}
}
struct StatePaths {
host: Arc<dyn TranscriptHost>,
transport: Arc<dyn SyncTransport>,
files: StateFileNames,
}
impl StatePaths {
fn env(&self, base: &str) -> PathBuf {
self.host.data_file(&self.transport.state_file_name(base))
}
fn root(&self, base: &str) -> PathBuf {
self.host.data_file(base)
}
fn acknowledged_scope(&self) -> Result<Option<String>, String> {
let ledger = read_scope(&self.env(&self.files.ledger), MAX_LEDGER_FILE_BYTES)
.map_err(|error| format!("read cloud sync ledger scope: {error}"))?;
if ledger.is_some() || !self.transport.legacy_import() {
return Ok(ledger);
}
read_scope(
&self.root(&self.files.legacy_acks),
MAX_LEGACY_ACKS_FILE_BYTES,
)
.map_err(|error| format!("read legacy acknowledgement scope: {error}"))
}
fn anchor_for(&self, binding: &ConsentScopeBinding) -> Result<Option<String>, String> {
if binding.authorized_scope.is_some() && binding.legacy_scope.is_some() {
return Ok(None);
}
self.acknowledged_scope()
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct TranscriptSyncState {
v: u8,
#[serde(default)]
sync_enabled: bool,
#[serde(default)]
epoch: u64,
#[serde(flatten)]
scope: ConsentScopeBinding,
}
impl Default for TranscriptSyncState {
fn default() -> Self {
Self {
v: 1,
sync_enabled: false,
epoch: 0,
scope: ConsentScopeBinding::default(),
}
}
}
fn fence_file_name(consent_file: &str) -> String {
match consent_file.rsplit_once('.') {
Some((stem, ext)) if !stem.is_empty() && !ext.contains(['/', '\\']) => {
format!("{stem}.fence.{ext}")
}
_ => format!("{consent_file}.fence"),
}
}
enum ConsentRead {
Missing,
Malformed,
Valid(TranscriptSyncState),
}
#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
enum FenceState {
Revoked,
Enabled,
}
#[derive(Serialize, Deserialize)]
struct SyncFence {
v: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
state: Option<FenceState>,
epoch: u64,
}
enum FenceRead {
Missing,
Malformed,
Valid(FenceState, u64),
}
struct ConsentStore {
host: Arc<dyn TranscriptHost>,
file: String,
fence_file: String,
revocation_incomplete: AtomicBool,
stop_unverified: AtomicBool,
disable_requests: AtomicU64,
stop_gate: Mutex<()>,
seen_epoch: AtomicU64,
outbox_file: PathBuf,
outbox_lock: Arc<Mutex<()>>,
account_changed_message: String,
lock: Mutex<()>,
}
impl ConsentStore {
fn path(&self) -> PathBuf {
self.host.data_file(&self.file)
}
fn fence_path(&self) -> PathBuf {
self.host.data_file(&self.fence_file)
}
fn load_fence(&self) -> Result<FenceRead, String> {
match read_capped(&self.fence_path(), MAX_FENCE_FILE_BYTES) {
Ok(bytes) => Ok(match serde_json::from_slice::<SyncFence>(&bytes) {
Ok(fence) => {
let state = match (fence.v, fence.state) {
(1, None) => Some(FenceState::Revoked),
(2, Some(state)) => Some(state),
_ => None,
};
match state {
Some(state) => {
self.seen_epoch.fetch_max(fence.epoch, Ordering::AcqRel);
FenceRead::Valid(state, fence.epoch)
}
None => FenceRead::Malformed,
}
}
Err(_) => FenceRead::Malformed,
}),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(FenceRead::Missing),
Err(error) => Err(format!("read cloud sync fence: {error}")),
}
}
fn write_fence(&self, state: FenceState, epoch: u64) -> Result<(), String> {
let bytes = serde_json::to_vec(&SyncFence {
v: 2,
state: Some(state),
epoch,
})
.map_err(|err| format!("serialize cloud sync fence: {err}"))?;
replace_file_atomically(&self.fence_path(), &bytes)
.map_err(|err| format!("write cloud sync fence: {err}"))
}
fn consent_trusted(&self, state: &TranscriptSyncState) -> bool {
if self.revocation_incomplete.load(Ordering::Acquire) {
return false;
}
matches!(
self.load_fence(),
Ok(FenceRead::Valid(FenceState::Enabled, fence)) if state.epoch == fence
)
}
fn read_trusted(&self) -> TranscriptSyncState {
let mut state = self.read();
if state.sync_enabled && !self.consent_trusted(&state) {
state.sync_enabled = false;
}
state
}
fn load(&self) -> Result<ConsentRead, String> {
match read_capped(&self.path(), MAX_CONSENT_FILE_BYTES) {
Ok(bytes) => Ok(match serde_json::from_slice(&bytes) {
Ok(state) => ConsentRead::Valid(state),
Err(_) => ConsentRead::Malformed,
}),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(ConsentRead::Missing),
Err(error) => Err(format!("read cloud sync state: {error}")),
}
}
fn read_checked(&self) -> Result<TranscriptSyncState, String> {
match self.load()? {
ConsentRead::Valid(state) => Ok(state),
ConsentRead::Missing => Ok(TranscriptSyncState::default()),
ConsentRead::Malformed => Err("cloud sync state is not valid".to_string()),
}
}
fn read(&self) -> TranscriptSyncState {
self.read_checked().unwrap_or_default()
}
fn transition_guard(&self) -> Result<std::sync::MutexGuard<'_, ()>, String> {
self.lock
.lock()
.map_err(|_| "cloud sync scope state lock poisoned".to_string())
}
fn recovery_state(&self) -> Result<Option<TranscriptSyncState>, String> {
let _outbox = self
.outbox_lock
.lock()
.map_err(|_| "deletion outbox lock poisoned".to_string())?;
let high = Outbox::peek_high_water(&self.outbox_file)?;
Ok(match self.load()? {
ConsentRead::Valid(state)
if state.sync_enabled && state.epoch >= high && self.consent_trusted(&state) =>
{
Some(state)
}
_ => None,
})
}
fn commit_recovery_if_authorized(
&self,
scheduler: &Scheduler,
receipt: &crate::recovery_marker::ReplayReceipt,
consent: &crate::recovery_marker::RecoveryConsent,
commit: &mut dyn FnMut() -> Result<(), String>,
) -> Result<bool, String> {
let _transition = self.transition_guard()?;
let Some(state) = self.recovery_state()? else {
return Ok(false);
};
if !receipt.cursor_committed
|| receipt.scope != consent.scope
|| state.epoch != consent.epoch
|| state.scope.authorized_scope.as_deref() != Some(consent.scope.as_str())
{
return Ok(false);
}
{
let _gate = self
.stop_gate
.lock()
.map_err(|_| "cloud sync stop gate poisoned".to_string())?;
if scheduler.consent_generation() != receipt.consent_generation
|| self.revocation_incomplete.load(Ordering::Acquire)
{
return Ok(false);
}
}
commit()?;
Ok(true)
}
fn bump(state: &mut TranscriptSyncState, floor: u64) -> Result<(), String> {
state.epoch = state
.epoch
.max(floor)
.checked_add(1)
.ok_or_else(|| "cloud sync consent epoch exhausted".to_string())?;
Ok(())
}
fn commit_transition(
&self,
state: &mut TranscriptSyncState,
revoke: bool,
write_consent: bool,
started: u64,
) -> Result<Vec<String>, String> {
let mut issues: Vec<String> = Vec::new();
let outbox_guard = match self.outbox_lock.lock() {
Ok(guard) => Some(guard),
Err(_) if revoke => {
issues.push("outbox left untouched: deletion outbox lock poisoned".to_string());
None
}
Err(_) => return Err("deletion outbox lock poisoned".to_string()),
};
let mut outbox = if outbox_guard.is_some() {
match Outbox::load(&self.outbox_file) {
Ok(outbox) => Some(outbox),
Err(error) if revoke => {
issues.push(format!("outbox left untouched: {error}"));
None
}
Err(error) => return Err(error),
}
} else {
None
};
let fence_floor = match self.load_fence() {
Ok(FenceRead::Valid(_, epoch)) => epoch,
_ => 0,
};
let floor = outbox
.as_ref()
.map_or(0, Outbox::high_water)
.max(fence_floor)
.max(self.seen_epoch.load(Ordering::Acquire));
let epoch_advanced = match Self::bump(state, floor) {
Ok(()) => true,
Err(error) if revoke => {
state.epoch = state.epoch.max(floor);
issues.push(error);
false
}
Err(error) => return Err(error),
};
let fence_error = match self.write_fence(FenceState::Revoked, state.epoch) {
Ok(()) => {
self.seen_epoch.fetch_max(state.epoch, Ordering::AcqRel);
if revoke {
self.revocation_incomplete.store(false, Ordering::Release);
} else {
let _gate = self.stop_gate.lock().unwrap_or_else(|e| e.into_inner());
if self.disable_requests.load(Ordering::Acquire) == started {
self.revocation_incomplete.store(false, Ordering::Release);
}
}
None
}
Err(error) if revoke => Some(error),
Err(error) => return Err(error),
};
if let Some(outbox) = outbox.as_mut().filter(|_| epoch_advanced) {
outbox.note_epoch(state.epoch);
if let Err(error) = outbox.save(&self.outbox_file) {
if revoke {
issues.push(error);
} else {
return Err(error);
}
}
}
if write_consent && epoch_advanced {
if let Err(error) = self.write_locked(state) {
if revoke {
issues.push(error);
} else {
return Err(error);
}
}
}
match fence_error {
Some(error) => {
issues.insert(0, format!("durable revocation incomplete: {error}"));
Err(issues.join("; "))
}
None => Ok(issues),
}
}
fn finish_enable(&self, epoch: u64) -> Result<(), String> {
self.write_fence(FenceState::Enabled, epoch)?;
self.seen_epoch.fetch_max(epoch, Ordering::AcqRel);
Ok(())
}
fn commit_enabled_transition(&self, state: &mut TranscriptSyncState) -> Result<(), String> {
let started = self.disable_requests.load(Ordering::Acquire);
self.commit_transition(state, false, true, started)?;
if self.disable_requests.load(Ordering::Acquire) != started {
return Err("cloud sync was turned off during the change".to_string());
}
self.finish_enable(state.epoch)
}
fn write_locked(&self, state: &TranscriptSyncState) -> Result<(), String> {
let bytes = serde_json::to_vec_pretty(state)
.map_err(|err| format!("serialize cloud sync state: {err}"))?;
replace_file_atomically(&self.path(), &bytes)
.map_err(|err| format!("commit cloud sync state: {err}"))
}
fn enabled(&self) -> bool {
self.read_trusted().sync_enabled
}
fn authorized_scope(&self) -> Option<String> {
self.read().scope.authorized_scope
}
fn authorize_live_scope(
&self,
actual: &str,
explicit: bool,
acknowledged: &dyn Fn(&ConsentScopeBinding) -> Result<Option<String>, String>,
) -> Result<(), String> {
let _guard = self.transition_guard()?;
let mut state = self.read();
if !explicit && !(state.sync_enabled && self.consent_trusted(&state)) {
return Err("cloud sync is disabled".to_string());
}
let before = state.clone();
let acknowledged = acknowledged(&state.scope)?;
let outcome = sync_core::authorize_scope(
&mut state.scope.authorized_scope,
&mut state.scope.legacy_scope,
acknowledged.as_deref(),
actual,
explicit,
&self.account_changed_message,
);
if state != before {
if state.scope.authorized_scope != before.scope.authorized_scope {
self.commit_enabled_transition(&mut state)?;
} else {
self.write_locked(&state)?;
}
}
outcome
}
}
struct ConsentAuth {
inner: Arc<dyn AuthProvider>,
consent: Arc<ConsentStore>,
paths: Arc<StatePaths>,
scheduler: Arc<Scheduler>,
scope_unavailable_message: String,
}
impl AuthProvider for ConsentAuth {
fn enabled(&self) -> bool {
self.consent.enabled() && self.inner.enabled()
}
fn availability(&self) -> SyncResult<()> {
self.inner.availability()?;
let live = self
.inner
.account_scope()
.ok_or_else(|| SyncError::Unavailable(self.scope_unavailable_message.clone()))?;
self.consent
.authorize_live_scope(&live, false, &|binding| self.paths.anchor_for(binding))
.map_err(SyncError::Unavailable)
}
fn generation(&self) -> u64 {
self.inner
.generation()
.wrapping_add(self.scheduler.consent_generation())
}
fn account_scope(&self) -> Option<String> {
self.inner.account_scope()
}
}
pub(crate) enum PassTurn {
Done(Option<PassReport>),
NotBefore(Duration),
}
struct PassOutbox {
stale: Vec<String>,
lift: Vec<String>,
keep: Vec<String>,
unproven: BTreeSet<String>,
eligible: BTreeSet<String>,
held: BTreeSet<String>,
epoch: Option<u64>,
}
struct PassObserver {
inner: Arc<dyn SyncObserver>,
deferred_uploads: Arc<AtomicUsize>,
}
impl SyncObserver for PassObserver {
fn event(&self, name: &'static str, detail: &str) {
if name == "uploads_deferred" {
if let Ok(count) = detail.trim().parse::<usize>() {
self.deferred_uploads.store(count, Ordering::Release);
}
}
self.inner.event(name, detail);
}
}
struct ExpectedScope<'a>(&'a LedgerGuard);
impl Drop for ExpectedScope<'_> {
fn drop(&mut self) {
self.0.set_expected(None);
}
}
struct Settled {
waiting: usize,
promoted: usize,
unverified: usize,
other_accounts: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DeletionBacklog {
pub unverified: usize,
pub other_accounts: usize,
pub waiting: usize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StatusReport {
Disabled,
Ready,
RevocationIncomplete,
Syncing,
Partial {
passes: u32,
downloaded: u64,
uploaded: u64,
},
Paused {
reason: &'static str,
passes: u32,
downloaded: u64,
uploaded: u64,
},
FeedStalled {
reason: &'static str,
passes: u32,
downloaded: u64,
uploaded: u64,
},
AccountChanged(String),
NotConfirmed(&'static str),
Unavailable(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DeleteOutcome {
Nothing,
Synced(u32),
Incomplete(&'static str),
Deferred,
}
struct KnownIds {
ids: Vec<String>,
zone_listed: bool,
}
fn outcome_tag(outcome: &PassOutcome) -> &'static str {
match outcome {
PassOutcome::Synced => "synced",
PassOutcome::Disabled => "disabled",
PassOutcome::Pending(_) => "pending",
PassOutcome::Deferred { .. } => "deferred",
PassOutcome::Cancelled => "cancelled",
PassOutcome::ConsentRequired(_) => "consent_required",
PassOutcome::Unavailable(_) => "unavailable",
PassOutcome::Failed(_) => "failed",
}
}
fn pass_summary(report: &PassReport) -> String {
let retry = match report.retry_after_secs {
Some(secs) if secs.is_finite() => format!("{secs:.0}"),
_ => "none".to_string(),
};
format!(
"outcome={} uploaded={} downloaded={} tombstones_applied={} tombstones_written={} quarantined={} blocked_reuploads={} retry_after_secs={retry}",
outcome_tag(&report.outcome),
report.uploaded,
report.downloaded,
report.tombstones_applied,
report.tombstones_written,
report.quarantined,
report.blocked_reuploads,
)
}
pub struct TranscriptSync {
profile: TranscriptSyncProfile,
local_source: String,
max_history_rows: usize,
event_sink: fn(&str, &str),
scope_unavailable_message: String,
host: Arc<dyn TranscriptHost>,
transport: Arc<dyn SyncTransport>,
paths: Arc<StatePaths>,
consent: Arc<ConsentStore>,
pub(crate) scheduler: Arc<Scheduler>,
engine: Mutex<Option<Hosted>>,
guard: Arc<LedgerGuard>,
window: Arc<RemoteWindow>,
deferred_uploads: Arc<AtomicUsize>,
sync_lock: tokio::sync::Mutex<()>,
outbox_lock: Arc<Mutex<()>>,
feed_watch: Mutex<FeedWatch>,
not_before: Mutex<Option<Instant>>,
suppressed: Arc<Mutex<BTreeSet<String>>>,
last_pass: Mutex<Option<PassRecord>>,
run_progress: Mutex<RunProgress>,
recovery_flag: Mutex<Option<String>>,
}
struct Hosted {
engine: SyncEngine<TranscriptMapper>,
local: Arc<HistoryLocalStore>,
}
#[derive(Debug, Clone, Default)]
struct RunProgress {
passes: u32,
downloaded: u64,
uploaded: u64,
}
enum LiveAccount {
Authorized(String),
Changed,
Unconfirmed,
}
impl LiveAccount {
fn tag(&self) -> &'static str {
match self {
Self::Authorized(_) => "authorized",
Self::Changed => "changed",
Self::Unconfirmed => "unconfirmed",
}
}
}
#[derive(Debug, Clone)]
struct PassRecord {
ledger_scope: Option<String>,
synced: bool,
partial: Option<RunProgress>,
consent_generation: u64,
halt: Option<&'static str>,
paused: Option<&'static str>,
}
struct PassResult {
paused: Option<&'static str>,
halt: Option<&'static str>,
report: PassReport,
ledger_scope: Option<String>,
partial: bool,
}
impl TranscriptSync {
pub fn new(
config: TranscriptSyncConfig,
host: Arc<dyn TranscriptHost>,
transport: Arc<dyn SyncTransport>,
) -> Result<Arc<Self>, String> {
config.validate()?;
let TranscriptSyncConfig {
profile,
consent_file,
local_source,
account_changed_message,
scope_unavailable_message,
max_history_rows,
scheduler,
event_sink,
spawn,
} = config;
let paths = Arc::new(StatePaths {
host: Arc::clone(&host),
transport: Arc::clone(&transport),
files: profile.files.clone(),
});
let outbox_lock = Arc::new(Mutex::new(()));
let consent = Arc::new(ConsentStore {
host: Arc::clone(&host),
fence_file: fence_file_name(&consent_file),
revocation_incomplete: AtomicBool::new(false),
stop_unverified: AtomicBool::new(false),
disable_requests: AtomicU64::new(0),
stop_gate: Mutex::new(()),
seen_epoch: AtomicU64::new(0),
file: consent_file,
outbox_file: paths.env(&profile.files.outbox),
outbox_lock: Arc::clone(&outbox_lock),
account_changed_message,
lock: Mutex::new(()),
});
let guard = Arc::new(LedgerGuard::new(paths.env(&profile.files.ledger)));
Ok(Arc::new(Self {
profile,
local_source,
max_history_rows,
event_sink,
scope_unavailable_message,
host,
transport,
paths,
consent,
scheduler: Arc::new(Scheduler::new(scheduler, spawn)),
engine: Mutex::new(None),
guard,
window: Arc::new(RemoteWindow::new()),
deferred_uploads: Arc::new(AtomicUsize::new(0)),
sync_lock: tokio::sync::Mutex::new(()),
outbox_lock,
feed_watch: Mutex::new(FeedWatch::default()),
not_before: Mutex::new(None),
suppressed: Arc::new(Mutex::new(BTreeSet::new())),
last_pass: Mutex::new(None),
run_progress: Mutex::new(RunProgress::default()),
recovery_flag: Mutex::new(None),
}))
}
pub(crate) fn emit(&self, name: &str, detail: &str) {
(self.event_sink)(name, &super::safe_detail(detail));
}
pub(crate) fn busy_reason(&self) -> Option<&'static str> {
self.host.busy_reason()
}
pub(crate) fn native_observer(&self) -> Option<Arc<dyn NativeObserver>> {
self.transport.native_observer()
}
pub(crate) fn notify_downloaded(&self, count: u32) {
self.host.downloaded(count);
}
pub fn sync_enabled(&self) -> bool {
self.consent.enabled()
}
pub fn authorized_scope(&self) -> Option<String> {
self.consent.authorized_scope()
}
pub fn has_local_choice(&self) -> bool {
self.consent.path().exists()
}
pub fn consent_generation(&self) -> u64 {
self.scheduler.consent_generation()
}
pub fn recovery_consent(&self) -> Option<crate::recovery_marker::RecoveryConsent> {
let _guard = self.consent.transition_guard().ok()?;
let state = self.consent.recovery_state().ok()??;
Some(crate::recovery_marker::RecoveryConsent {
scope: state.scope.authorized_scope?,
epoch: state.epoch,
})
}
pub fn commit_recovery_if_authorized(
&self,
receipt: &crate::recovery_marker::ReplayReceipt,
consent: &crate::recovery_marker::RecoveryConsent,
commit: &mut dyn FnMut() -> Result<(), String>,
) -> Result<bool, String> {
self.consent
.commit_recovery_if_authorized(&self.scheduler, receipt, consent, commit)
}
pub fn notify_local_change(&self) {
self.scheduler.request(PUSH);
}
pub fn notify_remote_change(&self) {
self.scheduler.request(PULL);
}
pub(crate) fn notify_reconcile(&self) {
self.scheduler.request(PUSH | PULL);
}
pub fn wake(&self) {
self.scheduler.wake();
}
pub fn start(self: &Arc<Self>) {
self.scheduler.start(self);
}
pub fn shutdown(&self) {
{
let _gate = self
.consent
.stop_gate
.lock()
.unwrap_or_else(|e| e.into_inner());
self.scheduler.stop();
}
if self.transport.set_enabled(false).is_err() {
self.consent.stop_unverified.store(true, Ordering::Release);
}
if let Some(native) = self.transport.native_observer() {
native.set_enabled(false);
}
}
fn outbox_path(&self) -> PathBuf {
self.paths.env(&self.profile.files.outbox)
}
fn scope_holds(&self, scope: &str, generation: u64) -> bool {
let state = self.consent.read_trusted();
state.sync_enabled
&& state.scope.authorized_scope.as_deref() == Some(scope)
&& self.scheduler.consent_generation() == generation
}
fn reobserve_holds(&self, scope: &str, generation: u64) -> bool {
self.scope_holds(scope, generation)
&& matches!(self.observe_live_scope(), LiveAccount::Authorized(live) if live == scope)
&& self.scope_holds(scope, generation)
}
fn capture_delete_token(&self) -> Result<DeleteToken, DeleteOutcome> {
let generation = self.scheduler.consent_generation();
let epoch = self.consent.read().epoch;
match self.observe_live_scope() {
LiveAccount::Authorized(scope) if self.scope_holds(&scope, generation) => {
Ok(DeleteToken {
scope,
generation,
epoch,
})
}
LiveAccount::Authorized(_) => Err(DeleteOutcome::Incomplete("scope_changed")),
LiveAccount::Changed => Err(DeleteOutcome::Incomplete("account_changed")),
LiveAccount::Unconfirmed => Err(DeleteOutcome::Incomplete("account_unconfirmed")),
}
}
pub(crate) fn admission_wait(&self) -> Option<Duration> {
let mut slot = self
.not_before
.lock()
.unwrap_or_else(PoisonError::into_inner);
let until = (*slot)?;
match until.checked_duration_since(Instant::now()) {
Some(remaining) if !remaining.is_zero() => Some(remaining),
_ => {
*slot = None;
None
}
}
}
fn set_not_before(&self, wait: Duration) {
let now = Instant::now();
let target = now
.checked_add(wait)
.or_else(|| now.checked_add(Duration::from_secs(FEED_BACKOFF_MAX_SECS)))
.unwrap_or(now);
let mut slot = self
.not_before
.lock()
.unwrap_or_else(PoisonError::into_inner);
if slot.map_or(true, |held| target > held) {
*slot = Some(target);
}
}
fn reset_feed_watch(&self) {
if let Ok(mut watch) = self.feed_watch.lock() {
*watch = FeedWatch::default();
}
*self
.not_before
.lock()
.unwrap_or_else(PoisonError::into_inner) = None;
self.scheduler.wake();
}
pub fn enqueue_deleted_ids(&self, ids: &[String]) -> Result<(), String> {
let before = self.consent.read_trusted();
let stamp = if !ids.is_empty() && before.sync_enabled {
match self.observe_live_scope() {
LiveAccount::Authorized(live) => Some((live, before.epoch)),
LiveAccount::Changed | LiveAccount::Unconfirmed => None,
}
} else {
None
};
self.enqueue_scoped(ids, stamp, false).map(|_| ())
}
fn enqueue_scoped(
&self,
ids: &[String],
stamp: Option<(String, u64)>,
strict: bool,
) -> Result<bool, String> {
if ids.is_empty() {
return Ok(true);
}
let guard = self
.outbox_lock
.lock()
.map_err(|_| "deletion outbox lock poisoned".to_string())?;
let path = self.outbox_path();
let mut outbox = Outbox::load(&path)?;
let state = self.trusted_state(&outbox).unwrap_or(None);
let holds = match (&state, &stamp) {
(Some(state), Some((scope, epoch))) => {
state.sync_enabled
&& state.scope.authorized_scope.as_deref() == Some(scope.as_str())
&& state.epoch == *epoch
}
_ => false,
};
if !holds && strict {
return Ok(false);
}
if let Some(state) = &state {
outbox.fence(state.epoch);
outbox.note_epoch(state.epoch);
}
let now = self.host.now_ms();
for raw in ids {
let id = self.profile.logical_cloud_id(raw);
match stamp.as_ref().filter(|_| holds) {
Some((scope, epoch)) => outbox.add(Some(scope.as_str()), *epoch, &id, now)?,
None => outbox.add(None, 0, &id, now)?,
}
}
outbox.save(&path)?;
drop(guard);
if stamp.is_some() && !holds {
self.emit(
"cloud_sync_deletion_unverified",
"reason=consent_epoch_moved",
);
}
self.notify_local_change();
Ok(holds)
}
fn trusted_state(&self, outbox: &Outbox) -> Result<Option<TranscriptSyncState>, String> {
Ok(match self.consent.load()? {
ConsentRead::Valid(state)
if state.epoch >= outbox.high_water() && self.consent.consent_trusted(&state) =>
{
Some(state)
}
_ => None,
})
}
fn fence_in_memory(&self, outbox: &mut Outbox) {
if let Ok(Some(state)) = self.trusted_state(outbox) {
outbox.fence(state.epoch);
}
}
fn prepare_pass_outbox(
&self,
live: Option<&str>,
ledger_scope: Option<&str>,
pending: &BTreeSet<String>,
) -> Result<PassOutbox, String> {
let _guard = self
.outbox_lock
.lock()
.map_err(|_| "deletion outbox lock poisoned".to_string())?;
let path = self.outbox_path();
let mut outbox = Outbox::load(&path)?;
let state = self.trusted_state(&outbox)?;
let proof: Option<(&str, u64)> = match (&state, ledger_scope) {
(Some(state), Some(scope))
if state.sync_enabled && state.scope.authorized_scope.as_deref() == Some(scope) =>
{
Some((scope, state.epoch))
}
_ => None,
};
let stale: Vec<String> = pending
.iter()
.filter(|id| {
!proof.is_some_and(|(scope, epoch)| outbox.handed_at(scope, epoch, id.as_str()))
})
.cloned()
.collect();
let (lift, keep): (Vec<String>, Vec<String>) = stale.iter().cloned().partition(|id| {
ledger_scope.is_some_and(|scope| outbox.handed_fresh(scope, id.as_str()))
});
let mut changed = false;
if !stale.is_empty() {
let now = self.host.now_ms();
for id in &stale {
outbox.add(None, 0, id, now)?;
}
let keep_set: BTreeSet<String> = keep.iter().cloned().collect();
if let Some(scope) = ledger_scope {
let marked_in = state
.as_ref()
.map_or_else(|| outbox.high_water(), |state| state.epoch);
outbox.mark_unproven(scope, marked_in, &keep_set);
}
changed = true;
}
changed |= outbox.retain_handed(|id| pending.contains(id));
let mut epoch = None;
let mut eligible = BTreeSet::new();
let mut held = BTreeSet::new();
let mut unproven = BTreeSet::new();
let mut fenced = 0;
if let Some(state) = &state {
fenced = outbox.fence(state.epoch);
changed |= fenced > 0;
changed |= outbox.note_epoch(state.epoch);
if let Some(live) = live {
if state.sync_enabled && state.scope.authorized_scope.as_deref() == Some(live) {
let queued = pending.len().saturating_sub(stale.len());
let budget = MAX_DRAIN_PER_PASS.saturating_sub(queued);
eligible = outbox.eligible(live, state.epoch, budget);
held = outbox.held_ids(live, state.epoch);
unproven = outbox.unproven_among(live, &eligible);
epoch = Some(state.epoch);
}
}
}
if changed {
outbox.save(&path)?;
}
if fenced > 0 {
self.emit(
"cloud_sync_deletions_fenced",
&format!("count={fenced} reason=consent_epoch_moved"),
);
}
if !stale.is_empty() {
self.emit(
"cloud_sync_pending_quarantined",
&format!(
"count={} kept_local={} reason={}",
stale.len(),
keep.len(),
if state.is_some() {
"no_current_epoch_evidence"
} else {
"consent_epoch_untrusted"
}
),
);
}
Ok(PassOutbox {
stale,
lift,
keep,
unproven,
eligible,
held,
epoch,
})
}
fn record_handover(
&self,
scope: &str,
epoch: u64,
ids: &BTreeSet<String>,
fresh: &BTreeSet<String>,
) -> Result<bool, String> {
if ids.is_empty() {
return Ok(true);
}
let _guard = self
.outbox_lock
.lock()
.map_err(|_| "deletion outbox lock poisoned".to_string())?;
let path = self.outbox_path();
let mut outbox = Outbox::load(&path)?;
let holds = self.trusted_state(&outbox)?.is_some_and(|state| {
state.sync_enabled
&& state.scope.authorized_scope.as_deref() == Some(scope)
&& state.epoch == epoch
});
if !holds {
self.emit(
"cloud_sync_drain_deferred",
&format!("count={} reason=consent_epoch_moved", ids.len()),
);
return Ok(false);
}
outbox.hand_over(scope, epoch, ids, fresh)?;
outbox.save(&path)?;
Ok(true)
}
fn settle_outbox(
&self,
scope: &str,
acknowledged: &BTreeSet<String>,
proven: &BTreeSet<String>,
) -> Result<Settled, String> {
let _guard = self
.outbox_lock
.lock()
.map_err(|_| "deletion outbox lock poisoned".to_string())?;
let path = self.outbox_path();
let mut outbox = Outbox::load(&path)?;
let mut changed = outbox.remove(scope, acknowledged);
let mut promoted = 0;
if !proven.is_empty() {
if let Ok(Some(state)) = self.trusted_state(&outbox) {
promoted = outbox.promote(scope, state.epoch, proven);
}
}
changed |= promoted > 0;
if changed {
outbox.save(&path)?;
}
self.fence_in_memory(&mut outbox);
Ok(Settled {
waiting: outbox.count_for(scope),
promoted,
unverified: outbox.count_unverified(),
other_accounts: outbox.count_other_accounts(Some(scope)),
})
}
fn queued_deletions(&self, scope: &str) -> usize {
let Ok(_guard) = self.outbox_lock.lock() else {
return 1;
};
Outbox::load(&self.outbox_path()).map_or(1, |mut outbox| {
self.fence_in_memory(&mut outbox);
outbox.count_for(scope)
})
}
pub fn deletion_backlog(&self) -> Result<DeletionBacklog, String> {
let _guard = self
.outbox_lock
.lock()
.map_err(|_| "deletion outbox lock poisoned".to_string())?;
let mut outbox = Outbox::load(&self.outbox_path())?;
self.fence_in_memory(&mut outbox);
let authorized = self.consent.authorized_scope();
Ok(DeletionBacklog {
unverified: outbox.count_unverified(),
other_accounts: outbox.count_other_accounts(authorized.as_deref()),
waiting: authorized
.as_deref()
.map_or(0, |scope| outbox.count_for(scope)),
})
}
fn observe_live_scope(&self) -> LiveAccount {
let Ok(auth) = self.transport.auth() else {
return LiveAccount::Unconfirmed;
};
if auth.availability().is_err() {
return LiveAccount::Unconfirmed;
}
let Some(live) = auth.account_scope().filter(|scope| !scope.is_empty()) else {
return LiveAccount::Unconfirmed;
};
let gate = self
.consent
.authorize_live_scope(&live, false, &|binding| self.paths.anchor_for(binding));
let authorized = self.consent.authorized_scope();
match (gate, authorized) {
(Ok(()), Some(scope)) if scope == live => LiveAccount::Authorized(live),
(_, Some(scope)) if scope != live => LiveAccount::Changed,
_ => LiveAccount::Unconfirmed,
}
}
fn legacy_files(&self, binding: &ConsentScopeBinding) -> LegacyFiles {
let scoped = |base: &str| match binding.authorized_scope.as_deref() {
Some(scope) => self.paths.root(&self.profile.legacy_scoped_file_name(
scope,
binding.legacy_scope.as_deref(),
base,
)),
None => self.paths.root(base),
};
let files = &self.profile.files;
LegacyFiles {
tombstones: scoped(&files.legacy_tombstones),
pending: scoped(&files.legacy_pending),
aliases: scoped(&files.legacy_aliases),
acks: self.paths.root(&files.legacy_acks),
}
}
fn migrate_legacy(&self, ledger_path: &Path, versions_path: &Path) -> Result<(), String> {
if !self.transport.legacy_import() || ledger_path.exists() {
return Ok(());
}
let auth = self.transport.auth()?;
auth.availability().map_err(|err| err.to_string())?;
let live = auth
.account_scope()
.filter(|scope| !scope.is_empty())
.ok_or_else(|| self.scope_unavailable_message.clone())?;
self.consent
.authorize_live_scope(&live, false, &|binding| self.paths.anchor_for(binding))?;
let binding = self.consent.read().scope;
if binding.authorized_scope.as_deref() != Some(live.as_str()) {
return Err(
"cloud sync authorized account does not match the live account".to_string(),
);
}
let migrated = self.profile.migrate_legacy_once(
&self.legacy_files(&binding),
Some(live.as_str()),
ledger_path,
versions_path,
)?;
if let Some(report) = migrated {
self.emit(
"cloud_sync_legacy_migrated",
&format!(
"tombstones={} pending_tombstones={} alias_ids={} seeded_versions={} dropped_conflicts={}",
report.tombstones,
report.pending_tombstones,
report.alias_ids,
report.seeded_versions,
report.dropped_conflicts
),
);
}
Ok(())
}
fn build_engine(&self) -> Result<Hosted, String> {
let ledger_path = self.paths.env(&self.profile.files.ledger);
let versions_path = self.paths.env(&self.profile.files.versions);
LocalVersions::load_capped(
versions_path.clone(),
MAX_VERSIONS_FILE_BYTES,
MAX_VERSIONS_ENTRIES,
)?;
self.migrate_legacy(&ledger_path, &versions_path)?;
let versions = LocalVersions::load_capped(
versions_path,
MAX_VERSIONS_FILE_BYTES,
MAX_VERSIONS_ENTRIES,
)?;
let local = Arc::new(HistoryLocalStore::new(
Arc::clone(&self.host),
self.profile.clone(),
self.local_source.clone(),
self.max_history_rows,
self.event_sink,
versions,
Arc::clone(&self.suppressed),
));
let backend: Arc<dyn Backend> = Arc::new(WindowedBackend::new(
self.transport.backend()?,
Arc::clone(&self.window),
));
let observer: Arc<dyn SyncObserver> = Arc::new(PassObserver {
inner: self.transport.observer(),
deferred_uploads: Arc::clone(&self.deferred_uploads),
});
let engine = SyncEngineBuilder::new(
Arc::new(TranscriptMapper::new(self.profile.clone())),
backend,
local.clone(),
Box::new(BoundedLedgerStorage::new(
ledger_path,
Arc::clone(&self.guard),
)),
)
.auth(Arc::new(ConsentAuth {
inner: self.transport.auth()?,
consent: Arc::clone(&self.consent),
paths: Arc::clone(&self.paths),
scheduler: Arc::clone(&self.scheduler),
scope_unavailable_message: self.scope_unavailable_message.clone(),
}))
.observer(observer)
.local_page_size(LOCAL_WINDOW_ROWS)
.max_uploads_per_pass(MAX_UPLOADS_PER_PASS)
.config(SyncConfig::default())
.build()
.map_err(|err| format!("build cloud sync engine: {err}"))?;
Ok(Hosted { engine, local })
}
fn recovery_flag_name(&self) -> String {
let fresh = || format!("history-recovery-{}", self.host.now_ms());
match self.recovery_flag.lock() {
Ok(mut flag) => flag.get_or_insert_with(fresh).clone(),
Err(_) => fresh(),
}
}
fn clear_recovery_flag(&self) {
if let Ok(mut flag) = self.recovery_flag.lock() {
*flag = None;
}
}
fn run_engine_pass(
&self,
consent_generation: u64,
expect: Option<&DeleteToken>,
) -> Result<PassReport, String> {
let result = self.run_engine_pass_inner(consent_generation, expect);
let record = match &result {
Ok(done) => {
let synced = done.report.outcome == PassOutcome::Synced;
let halted = synced && done.halt.is_some();
let run = match self.run_progress.lock() {
Ok(mut progress) => {
if synced && (done.partial || halted) {
progress.passes = progress.passes.saturating_add(1);
progress.downloaded = progress.downloaded.saturating_add(
u64::try_from(done.report.downloaded).unwrap_or(u64::MAX),
);
progress.uploaded = progress.uploaded.saturating_add(
u64::try_from(done.report.uploaded).unwrap_or(u64::MAX),
);
Some(progress.clone())
} else {
if synced {
*progress = RunProgress::default();
}
None
}
}
Err(_) => (synced && (done.partial || halted)).then(RunProgress::default),
};
PassRecord {
ledger_scope: done.ledger_scope.clone(),
synced,
partial: run,
consent_generation,
halt: if synced { done.halt } else { None },
paused: if synced { done.paused } else { None },
}
}
Err(_) => PassRecord {
ledger_scope: None,
synced: false,
partial: None,
consent_generation,
halt: None,
paused: None,
},
};
if let Ok(mut slot) = self.last_pass.lock() {
*slot = Some(record);
}
result.map(|done| done.report)
}
fn run_engine_pass_inner(
&self,
consent_generation: u64,
expect: Option<&DeleteToken>,
) -> Result<PassResult, String> {
self.transport.prepare()?;
let mut slot = self
.engine
.lock()
.map_err(|_| "cloud sync engine lock poisoned".to_string())?;
let account = self.observe_live_scope();
self.emit(
"cloud_sync_live_account",
&format!("state={}", account.tag()),
);
let live: Option<String> = match account {
LiveAccount::Authorized(scope) => Some(scope),
LiveAccount::Changed | LiveAccount::Unconfirmed => None,
};
if let Some(token) = expect {
if live.as_deref() != Some(token.scope.as_str())
|| !self.scope_holds(&token.scope, token.generation)
{
return Err(PASS_TOKEN_MISMATCH.to_string());
}
}
let live = live.as_deref();
self.guard.set_expected(live.map(str::to_string));
let _expected = ExpectedScope(&self.guard);
let mut checked: Option<Disk> = if slot.is_some() {
Some(self.guard.check()?)
} else {
None
};
let rebuild = match &checked {
None => true,
Some(Disk::Matches(_)) => false,
Some(Disk::Differs) => {
self.emit("cloud_sync_ledger_rebuilt", "reason=disk_changed");
true
}
};
if rebuild {
*slot = None;
*slot = Some(self.build_engine()?);
checked = None;
}
let hosted = slot
.as_mut()
.ok_or_else(|| "cloud sync engine unavailable".to_string())?;
hosted.local.begin_pass();
self.window.begin_pass();
self.deferred_uploads.store(0, Ordering::Release);
let scope_before = hosted.engine.ledger().scope.clone();
let disk = match checked {
Some(disk) => disk,
None => self.guard.check()?,
};
let durable = match disk {
Disk::Matches(state) => Some(state),
Disk::Differs => None,
};
let drain_ok = match (live, durable.as_ref()) {
(Some(live), Some(ScopeState::Bound(on_disk))) => {
on_disk.as_str() == live && scope_before.as_deref() == Some(live)
}
_ => false,
};
let ledger_pending = hosted.engine.ledger().pending_tombstones.clone();
let prepared = self.prepare_pass_outbox(live, scope_before.as_deref(), &ledger_pending)?;
if !prepared.stale.is_empty() {
if durable.is_none() {
return Err(
"cloud sync ledger changed; queued deletions are not verified".to_string(),
);
}
hosted
.engine
.withdraw_pending_tombstones(&prepared.lift, &prepared.keep)
.map_err(|err| format!("withdraw cloud tombstones: {err}"))?;
}
let mut batch: BTreeSet<String> = BTreeSet::new();
if let Some(live) = live {
if drain_ok {
if let Some(epoch) = prepared.epoch {
let mut queue: BTreeSet<String> = BTreeSet::new();
let mut fresh: BTreeSet<String> = BTreeSet::new();
let mut unconfirmed: BTreeSet<String> = BTreeSet::new();
{
let ledger = hosted.engine.ledger();
for id in &prepared.eligible {
let buried = ledger.tombstones.contains_key(id);
let queued = ledger.pending_tombstones.contains(id);
let doubtful = prepared.unproven.contains(id);
if buried && !queued && !doubtful {
continue;
}
queue.insert(id.clone());
if doubtful {
unconfirmed.insert(id.clone());
}
if !buried {
fresh.insert(id.clone());
}
}
}
if self.record_handover(live, epoch, &queue, &fresh)? {
batch = prepared.eligible;
let ids: Vec<String> = queue.iter().cloned().collect();
hosted
.engine
.note_deleted_many(&ids, &unconfirmed)
.map_err(|err| format!("queue cloud tombstones: {err}"))?;
}
}
} else if !prepared.eligible.is_empty() {
let why = if scope_before.as_deref() == Some(live) {
"disk_unverified"
} else {
"ledger_not_bound"
};
self.emit(
"cloud_sync_drain_deferred",
&format!("count={} reason={why}", prepared.eligible.len()),
);
}
if let Ok(mut suppressed) = self.suppressed.lock() {
*suppressed = prepared.held;
}
}
let scope = self.consent.authorized_scope();
let recovery = scope
.as_deref()
.is_some_and(|scope| self.host.recovery_pending(scope));
if recovery {
let flag = self.recovery_flag_name();
hosted.engine.request_full_replay_once(&flag);
} else {
self.clear_recovery_flag();
}
self.activate_transport_for_pass(consent_generation)?;
let report = hosted.engine.run_pass();
let held = self
.suppressed
.lock()
.map(|mut suppressed| std::mem::take(&mut *suppressed))
.unwrap_or_default();
let (bound_now, acknowledged, proven) = {
let ledger = hosted.engine.ledger();
let bound_now = ledger.scope.clone();
let acknowledged: BTreeSet<String> = if drain_ok {
batch
.iter()
.filter(|id| {
ledger.tombstones.contains_key(*id)
&& !ledger.pending_tombstones.contains(*id)
})
.cloned()
.collect()
} else {
BTreeSet::new()
};
let proven: BTreeSet<String> = BTreeSet::new();
(bound_now, acknowledged, proven)
};
drop(held);
let metrics = hosted.engine.metrics();
self.emit(
"cloud_sync_pass",
&format!(
"{} remote_versions={} tombstones={} pending_uploads={} pending_tombstones={} pending_apply={}",
pass_summary(&report),
metrics.remote_versions,
metrics.tombstones,
metrics.pending_uploads,
metrics.pending_tombstones,
metrics.pending_apply
),
);
hosted.local.end_pass();
let window_totals = self.window.finish_pass();
if window_totals.stalled {
self.emit("cloud_sync_feed_stalled", "reason=no_progressing_cursor");
}
let feed_cut = window_totals.truncated;
let uploads_left = self.deferred_uploads.swap(0, Ordering::AcqRel);
drop(slot);
let mut drain_continues = false;
if let Some(live) = live {
if bound_now.as_deref() == Some(live) {
match self.settle_outbox(live, &acknowledged, &proven) {
Ok(settled) => {
if settled.promoted > 0 {
self.emit(
"cloud_sync_deletions_proven",
&format!("count={}", settled.promoted),
);
}
if settled.unverified > 0 || settled.other_accounts > 0 {
self.emit(
"cloud_sync_deletion_backlog",
&format!(
"unverified={} other_accounts={}",
settled.unverified, settled.other_accounts
),
);
}
let just_bound = !drain_ok && scope_before.as_deref() != Some(live);
drain_continues = settled.waiting > 0
&& (just_bound || !acknowledged.is_empty() || settled.promoted > 0);
}
Err(err) => self.emit("cloud_sync_outbox_settle_failed", &err),
}
}
}
let synced = report.outcome == PassOutcome::Synced;
let mut halt: Option<&'static str> = None;
if synced {
let facts = FeedFacts {
stalled: window_totals.stalled,
cut: feed_cut,
end_token: window_totals.end_token.as_deref(),
applied: report.downloaded > 0 || report.tombstones_applied > 0,
};
let (verdict, backoff) = match self.feed_watch.lock() {
Ok(mut watch) => {
let verdict = watch.judge(&facts);
(verdict, watch.backoff())
}
Err(_) => (
FeedVerdict::Halt("watch_unavailable"),
Duration::from_secs(FEED_BACKOFF_MAX_SECS),
),
};
if let FeedVerdict::Halt(reason) = verdict {
halt = Some(reason);
self.emit(
"cloud_sync_feed_halted",
&format!("reason={reason} backoff_secs={}", backoff.as_secs()),
);
self.set_not_before(backoff);
self.scheduler.request(PULL);
}
}
let partial = feed_cut || uploads_left > 0 || drain_continues;
let mut paused: Option<&'static str> = None;
if !partial || halt.is_some() {
if let Ok(mut watch) = self.feed_watch.lock() {
watch.immediate_streak = 0;
}
}
if partial {
self.emit(
"cloud_sync_pass_partial",
&format!(
"feed_cut={feed_cut} uploads_left={uploads_left} deletions_continue={drain_continues}"
),
);
let progressed = feed_cut
|| drain_continues
|| report.uploaded > 0
|| report.downloaded > 0
|| report.tombstones_written > 0
|| report.tombstones_applied > 0;
if synced && progressed && halt.is_none() {
let limit_taken = self
.feed_watch
.lock()
.map_or(true, |mut watch| watch.note_immediate());
if limit_taken {
paused = Some("pass_budget");
let wait = Duration::from_secs(FEED_BACKOFF_BASE_SECS);
self.emit(
"cloud_sync_pass_budget_pause",
&format!(
"limit={FEED_MAX_IMMEDIATE_PARTIAL_PASSES} backoff_secs={}",
wait.as_secs()
),
);
self.set_not_before(wait);
self.scheduler.request(PULL);
} else {
self.scheduler.request(PULL | PUSH);
}
}
}
if recovery && synced && !partial && halt.is_none() && paused.is_none() {
if let Some(scope) = scope {
self.host.recovery_complete(&scope, consent_generation)?;
}
self.clear_recovery_flag();
}
Ok(PassResult {
paused,
halt,
report,
ledger_scope: bound_now,
partial,
})
}
pub async fn pass(self: &Arc<Self>) -> Result<Option<PassReport>, String> {
self.pass_for(None).await
}
async fn pass_for(
self: &Arc<Self>,
expect: Option<DeleteToken>,
) -> Result<Option<PassReport>, String> {
Ok(match self.pass_gated(expect, false).await? {
PassTurn::Done(report) => report,
PassTurn::NotBefore(_) => None,
})
}
pub(crate) async fn pass_automatic(self: &Arc<Self>) -> Result<PassTurn, String> {
self.pass_gated(None, true).await
}
async fn pass_gated(
self: &Arc<Self>,
expect: Option<DeleteToken>,
automatic: bool,
) -> Result<PassTurn, String> {
if !self.consent.enabled() {
return Ok(PassTurn::Done(None));
}
let consent_generation = match &expect {
Some(token) => {
if !self.scope_holds(&token.scope, token.generation) {
return Err(PASS_TOKEN_MISMATCH.to_string());
}
token.generation
}
None => self.scheduler.consent_generation(),
};
if let Some(reason) = self.host.busy_reason() {
self.emit(
"cloud_sync_deferred",
&format!("phase=pass reason={reason}"),
);
self.notify_local_change();
return Ok(PassTurn::Done(None));
}
let _sync_guard = self.sync_lock.lock().await;
if let Some(reason) = self.host.busy_reason() {
self.emit(
"cloud_sync_deferred",
&format!("phase=pass reason={reason}_after_lock"),
);
self.notify_local_change();
return Ok(PassTurn::Done(None));
}
if automatic {
if let Some(remaining) = self.admission_wait() {
self.emit(
"cloud_sync_deferred",
"phase=pass reason=not_before_after_lock",
);
return Ok(PassTurn::NotBefore(remaining));
}
}
let this = Arc::clone(self);
let report = tokio::task::spawn_blocking(move || {
this.run_engine_pass(consent_generation, expect.as_ref())
})
.await
.map_err(|err| format!("cloud_sync_pass task join: {err}"))??;
if !self.consent.enabled() || self.scheduler.consent_generation() != consent_generation {
self.emit("cloud_sync_stale_completion_dropped", "phase=pass");
return Ok(PassTurn::Done(None));
}
Ok(PassTurn::Done(Some(report)))
}
pub async fn set_enabled(self: &Arc<Self>, enabled: bool) -> Result<(), String> {
let started = self.consent.disable_requests.load(Ordering::Acquire);
let mut early_stop: Result<(), String> = Ok(());
if !enabled {
{
let _gate = self
.consent
.stop_gate
.lock()
.unwrap_or_else(|e| e.into_inner());
self.consent.disable_requests.fetch_add(1, Ordering::AcqRel);
self.consent
.revocation_incomplete
.store(true, Ordering::Release);
self.scheduler.disable();
}
early_stop = self.transport.set_enabled(false);
if let Some(native) = self.transport.native_observer() {
native.set_enabled(false);
}
}
let _sync_guard = self.sync_lock.lock().await;
let stop_unproven: Option<String> = if !enabled {
let note = self.stop_transport_and_verify(std::mem::replace(&mut early_stop, Ok(())));
if let Some(native) = self.transport.native_observer() {
native.set_enabled(false);
}
note
} else {
None
};
let this = Arc::clone(self);
let joined = tokio::task::spawn_blocking(move || this.persist_consent(enabled, started))
.await
.map_err(|err| format!("join cloud_sync_set_enabled: {err}"));
let mut result = if enabled {
joined?
} else {
joined.and_then(|r| r)
};
if enabled && result.is_ok() {
enum Outcome {
Started,
Superseded,
ActivationFailed(String),
}
let outcome = {
let _gate = self
.consent
.stop_gate
.lock()
.unwrap_or_else(|e| e.into_inner());
if self.consent.disable_requests.load(Ordering::Acquire) != started {
Outcome::Superseded
} else {
match self.transport.set_enabled(true) {
Ok(()) => {
self.scheduler.enable();
Outcome::Started
}
Err(error) => Outcome::ActivationFailed(error),
}
}
};
match outcome {
Outcome::Started => {
self.consent.stop_unverified.store(false, Ordering::Release);
}
Outcome::Superseded => {
self.revoke_fence_best_effort();
result =
Err("cloud sync was turned off while it was being turned on".to_string());
}
Outcome::ActivationFailed(error) => {
self.emit("cloud_sync_activation_failed", "transport=not_started");
let stop_note = self.stop_transport_and_verify(Ok(()));
let this = Arc::clone(self);
let rolled_back =
tokio::task::spawn_blocking(move || this.persist_revocation())
.await
.map_err(|err| format!("join cloud_sync_rollback: {err}"))
.and_then(|r| r);
let detail = super::safe_detail(&error);
let message = match (stop_note, rolled_back.is_err()) {
(None, false) => format!(
"cloud sync could not be started and was turned back off: {detail}"
),
(None, true) => format!(
"cloud sync could not be started and is stopped, but turning it off could not be saved: {detail}"
),
(Some(note), false) => format!(
"cloud sync could not be started; it is off in settings but {note}: {detail}"
),
(Some(note), true) => format!(
"cloud sync could not be started; {note}, and turning it off could not be saved: {detail}"
),
};
result = Err(message);
}
}
}
if !enabled {
result = match (stop_unproven, result) {
(None, result) => result,
(Some(note), Ok(())) => Err(format!(
"cloud sync is off in settings, but {note}; sync stays off"
)),
(Some(note), Err(saved)) => Err(format!(
"{note}, and turning sync off could not be saved: {saved}"
)),
};
}
if result.is_ok() || !enabled {
if !enabled {
self.scheduler.disable();
}
if let Ok(mut slot) = self.last_pass.lock() {
*slot = None;
}
if let Ok(mut progress) = self.run_progress.lock() {
*progress = RunProgress::default();
}
self.clear_recovery_flag();
self.reset_feed_watch();
if let Some(native) = self.transport.native_observer() {
native.set_enabled(enabled);
}
}
result
}
fn persist_consent(&self, enabled: bool, started: u64) -> Result<(), String> {
if !enabled {
return self.persist_revocation();
}
let _guard = self.consent.transition_guard()?;
self.refuse_if_disable_requested(started)?;
if let Err(error) = self.consent.load() {
return Err(error);
}
Outbox::peek_high_water(&self.consent.outbox_file)?;
let actual = self.observe_account_for_enable()?;
match self.commit_enable(&actual, started) {
Ok(epoch) => self
.recheck_enable_admission(epoch, started)
.map_err(|error| self.with_transport_detail(error)),
Err(error) => Err(self.with_transport_detail(error)),
}
}
fn observe_account_for_enable(&self) -> Result<String, String> {
let auth = self.transport.auth()?;
auth.availability().map_err(|err| err.to_string())?;
auth.account_scope()
.filter(|scope| !scope.is_empty())
.ok_or_else(|| "cloud sync scope check returned no scope".to_string())
}
fn recheck_enable_admission(&self, epoch: u64, started: u64) -> Result<(), String> {
self.refuse_if_disable_requested(started)?;
let stale = || "cloud sync consent changed while it was being turned on".to_string();
match self.consent.load_fence() {
Ok(FenceRead::Valid(FenceState::Enabled, fence)) if fence == epoch => {}
_ => return Err(stale()),
}
match self.consent.load() {
Ok(ConsentRead::Valid(state)) if state.sync_enabled && state.epoch == epoch => Ok(()),
_ => Err(stale()),
}
}
fn activate_transport_for_pass(&self, consent_generation: u64) -> Result<(), String> {
if self.transport.confirmed_stopped() == Some(false) {
return Ok(());
}
let _gate = self
.consent
.stop_gate
.lock()
.unwrap_or_else(|e| e.into_inner());
if self.scheduler.consent_generation() != consent_generation
|| self.consent.revocation_incomplete.load(Ordering::Acquire)
{
return Err("cloud sync was turned off before the sync helper started".to_string());
}
self.transport
.set_enabled(true)
.map_err(|_| "the sync helper could not be started".to_string())
}
fn with_transport_detail(&self, error: String) -> String {
match self.stop_transport_after_refused_enable() {
Some(detail) => format!("{error}; {detail}"),
None => error,
}
}
fn refuse_if_disable_requested(&self, started: u64) -> Result<(), String> {
if self.consent.disable_requests.load(Ordering::Acquire) != started {
return Err("cloud sync was turned off while it was being turned on".to_string());
}
Ok(())
}
fn stop_transport_after_refused_enable(&self) -> Option<String> {
if self.consent.enabled() {
return None;
}
let set = self.transport.set_enabled(false);
if set.is_ok() && self.transport.confirmed_stopped() == Some(true) {
self.consent.stop_unverified.store(false, Ordering::Release);
return None;
}
self.consent.stop_unverified.store(true, Ordering::Release);
Some("the sync helper could not be confirmed stopped; sync stays off".to_string())
}
fn stop_transport_and_verify(&self, earlier: Result<(), String>) -> Option<String> {
let last = self.transport.set_enabled(false);
let proof = self.transport.confirmed_stopped();
let note = if earlier.is_err() || last.is_err() {
"a request to stop the sync helper failed, so the stop could not be confirmed"
} else if proof == Some(false) {
"the sync helper still reports running"
} else if proof.is_none() {
"the sync helper could not be confirmed stopped"
} else {
self.consent.stop_unverified.store(false, Ordering::Release);
return None;
};
self.consent.stop_unverified.store(true, Ordering::Release);
self.emit("cloud_sync_stop_unverified", "stop=not_proven");
Some(note.to_string())
}
fn revoke_fence_best_effort(&self) {
if let Ok(_guard) = self.consent.transition_guard() {
if let Ok(FenceRead::Valid(_, epoch)) = self.consent.load_fence() {
let _ = self.consent.write_fence(FenceState::Revoked, epoch);
}
}
}
fn commit_enable(&self, actual: &str, started: u64) -> Result<u64, String> {
let mut state = match self.consent.load()? {
ConsentRead::Valid(state) => state,
ConsentRead::Missing | ConsentRead::Malformed => TranscriptSyncState::default(),
};
let acknowledged = self.paths.anchor_for(&state.scope)?;
sync_core::authorize_scope(
&mut state.scope.authorized_scope,
&mut state.scope.legacy_scope,
acknowledged.as_deref(),
actual,
true,
&self.consent.account_changed_message,
)?;
state.sync_enabled = true;
self.refuse_if_disable_requested(started)?;
self.consent
.commit_transition(&mut state, false, true, started)?;
self.refuse_if_disable_requested(started)?;
self.consent.finish_enable(state.epoch)?;
Ok(state.epoch)
}
fn persist_revocation(&self) -> Result<(), String> {
self.consent
.revocation_incomplete
.store(true, Ordering::Release);
let _guard = self
.consent
.lock
.lock()
.unwrap_or_else(|error| error.into_inner());
let (mut state, write_consent) = match self.consent.load() {
Ok(ConsentRead::Valid(state)) => (state, true),
Ok(ConsentRead::Missing | ConsentRead::Malformed) => {
(TranscriptSyncState::default(), true)
}
Err(_) => (TranscriptSyncState::default(), false),
};
state.sync_enabled = false;
match self
.consent
.commit_transition(&mut state, true, write_consent, 0)
{
Ok(issues) => {
if !write_consent || !issues.is_empty() {
self.emit(
"cloud_sync_revocation_partial",
&format!(
"fence=written other_failures={}",
issues.len() + usize::from(!write_consent)
),
);
}
Ok(())
}
Err(error) => {
self.emit("cloud_sync_revocation_incomplete", "fence=not_written");
Err(error)
}
}
}
pub async fn status(self: &Arc<Self>) -> Result<StatusReport, String> {
let this = Arc::clone(self);
let check = tokio::task::spawn_blocking(move || this.status_blocking());
match tokio::time::timeout(STATUS_CHECK_TIMEOUT, check).await {
Ok(Ok(report)) => Ok(report),
Ok(Err(err)) => Err(format!("cloud_sync_status task join: {err}")),
Err(_) => Ok(StatusReport::NotConfirmed("check_timed_out")),
}
}
fn status_blocking(&self) -> StatusReport {
let revocation_incomplete = self.consent.revocation_incomplete.load(Ordering::Acquire);
let stop_unverified = self.consent.stop_unverified.load(Ordering::Acquire);
if revocation_incomplete && stop_unverified {
return StatusReport::NotConfirmed("stop_unverified_revocation_incomplete");
}
if revocation_incomplete {
return StatusReport::RevocationIncomplete;
}
if stop_unverified {
return StatusReport::NotConfirmed("stop_unverified");
}
let state = match self.consent.read_checked() {
Ok(state) => state,
Err(_) => return StatusReport::NotConfirmed("consent_unreadable"),
};
if !state.sync_enabled {
return StatusReport::Disabled;
}
match self.consent.load_fence() {
Ok(FenceRead::Valid(FenceState::Enabled, fence)) if state.epoch == fence => {}
Ok(_) => return StatusReport::NotConfirmed("consent_untrusted"),
Err(_) => return StatusReport::NotConfirmed("fence_unreadable"),
}
match Outbox::peek_high_water(&self.consent.outbox_file) {
Ok(high) if state.epoch >= high => {}
Ok(_) => return StatusReport::NotConfirmed("consent_epoch_invalid"),
Err(_) => return StatusReport::NotConfirmed("outbox_unreadable"),
}
let Some(authorized) = state.scope.authorized_scope else {
return StatusReport::NotConfirmed("not_authorized");
};
match self.engine.try_lock() {
Err(TryLockError::WouldBlock) => return StatusReport::Syncing,
Err(TryLockError::Poisoned(_)) => {
return StatusReport::NotConfirmed("engine_unavailable");
}
Ok(guard) => drop(guard),
}
let auth = match self.transport.auth() {
Ok(auth) => auth,
Err(detail) => {
return StatusReport::Unavailable(super::safe_detail(&detail));
}
};
if let Err(error) = auth.availability() {
return StatusReport::Unavailable(super::safe_detail(&error.to_string()));
}
let live = match auth.account_scope() {
Some(scope) if !scope.is_empty() => scope,
_ => {
return StatusReport::NotConfirmed("account_unknown");
}
};
if live != authorized {
if let Ok(mut slot) = self.last_pass.lock() {
*slot = None;
}
if let Ok(mut progress) = self.run_progress.lock() {
*progress = RunProgress::default();
}
return StatusReport::AccountChanged(self.consent.account_changed_message.clone());
}
match read_scope(
&self.paths.env(&self.profile.files.ledger),
MAX_LEDGER_FILE_BYTES,
) {
Err(_) => return StatusReport::NotConfirmed("ledger_unreadable"),
Ok(scope) if scope.as_deref() != Some(authorized.as_str()) => {
return StatusReport::NotConfirmed("ledger_unbound");
}
Ok(_) => {}
}
let Ok(record) = self.last_pass.lock() else {
return StatusReport::NotConfirmed("engine_unavailable");
};
match record.as_ref() {
Some(pass) if pass.consent_generation == self.scheduler.consent_generation() => {
if pass.synced && pass.ledger_scope.as_deref() == Some(authorized.as_str()) {
if let Some(reason) = pass.halt {
let run = pass.partial.clone().unwrap_or_default();
return StatusReport::FeedStalled {
reason,
passes: run.passes,
downloaded: run.downloaded,
uploaded: run.uploaded,
};
}
if let Some(reason) = pass.paused {
let run = pass.partial.clone().unwrap_or_default();
return StatusReport::Paused {
reason,
passes: run.passes,
downloaded: run.downloaded,
uploaded: run.uploaded,
};
}
match &pass.partial {
Some(run) => StatusReport::Partial {
passes: run.passes,
downloaded: run.downloaded,
uploaded: run.uploaded,
},
None => StatusReport::Ready,
}
} else {
StatusReport::NotConfirmed("last_pass_not_synced")
}
}
_ => {
self.scheduler.request(PULL);
StatusReport::NotConfirmed("no_pass_yet")
}
}
}
fn known_ids(&self, token: &DeleteToken) -> Result<KnownIds, String> {
let mut history = self.host.open_history()?;
let mut ids: BTreeSet<String> = BTreeSet::new();
scan_keys(&mut *history, self.max_history_rows, |key| {
ids.insert(self.profile.logical_cloud_id(&key.delivery_id));
})?;
let ledger = read_ledger_ids(&self.paths.env(&self.profile.files.ledger))
.map_err(|error| format!("read cloud sync ledger: {error}"))?;
let memory_scope: Option<String> = self
.engine
.lock()
.map_err(|_| "cloud sync engine lock poisoned".to_string())?
.as_ref()
.and_then(|hosted| hosted.engine.ledger().scope.clone());
let bound = Some(token.scope.clone());
let zone_listed = ledger.scope == bound && memory_scope == bound;
if zone_listed {
ids.extend(ledger.remote_ids);
}
Ok(KnownIds {
ids: ids.into_iter().collect(),
zone_listed,
})
}
pub async fn delete_everywhere(self: &Arc<Self>) -> Result<DeleteOutcome, String> {
if !self.consent.enabled() {
return Err("cloud sync is disabled".to_string());
}
let this = Arc::clone(self);
let token = match tokio::task::spawn_blocking(move || this.capture_delete_token())
.await
.map_err(|err| format!("cloud_sync_delete_everywhere task join: {err}"))?
{
Ok(token) => token,
Err(outcome) => return Ok(outcome),
};
let this = Arc::clone(self);
let listing = token.clone();
let KnownIds { ids, zone_listed } =
tokio::task::spawn_blocking(move || this.known_ids(&listing))
.await
.map_err(|err| format!("cloud_sync_delete_everywhere task join: {err}"))??;
if ids.is_empty() {
let this = Arc::clone(self);
let check = token.clone();
let holds = tokio::task::spawn_blocking(move || {
this.reobserve_holds(&check.scope, check.generation)
})
.await
.map_err(|err| format!("cloud_sync_delete_everywhere task join: {err}"))?;
return Ok(if !holds {
DeleteOutcome::Incomplete("scope_changed")
} else if zone_listed {
DeleteOutcome::Nothing
} else {
DeleteOutcome::Incomplete("zone_not_listed")
});
}
let count = u32::try_from(ids.len()).unwrap_or(u32::MAX);
let this = Arc::clone(self);
let writing = token.clone();
let queued = tokio::task::spawn_blocking(move || {
if !this.reobserve_holds(&writing.scope, writing.generation) {
return Ok(false);
}
this.enqueue_scoped(&ids, Some((writing.scope.clone(), writing.epoch)), true)
})
.await
.map_err(|err| format!("cloud_sync_delete_everywhere task join: {err}"))??;
if !queued {
return Ok(DeleteOutcome::Incomplete("scope_changed"));
}
self.reset_feed_watch();
let report = match self.pass_for(Some(token.clone())).await {
Ok(report) => report,
Err(err) if err == PASS_TOKEN_MISMATCH => {
return Ok(DeleteOutcome::Incomplete("scope_changed"));
}
Err(err) => return Err(err),
};
match report {
Some(report) if report.outcome == PassOutcome::Synced => {
let (partial, halted, paused, same_scope) = self
.last_pass
.lock()
.ok()
.and_then(|slot| {
slot.as_ref().map(|record| {
(
record.partial.is_some(),
record.halt.is_some(),
record.paused.is_some(),
record.consent_generation == token.generation
&& record.ledger_scope.as_deref() == Some(token.scope.as_str()),
)
})
})
.unwrap_or((true, false, false, true));
if !same_scope {
return Ok(DeleteOutcome::Incomplete("scope_changed"));
}
if halted {
return Ok(DeleteOutcome::Incomplete("feed_stalled"));
}
if paused {
return Ok(DeleteOutcome::Incomplete("paused"));
}
if self.queued_deletions(&token.scope) > 0 {
return Ok(DeleteOutcome::Incomplete("queued"));
}
if partial {
return Ok(DeleteOutcome::Incomplete("partial"));
}
let this = Arc::clone(self);
let check = token.clone();
let holds = tokio::task::spawn_blocking(move || {
this.reobserve_holds(&check.scope, check.generation)
})
.await
.map_err(|err| format!("cloud_sync_delete_everywhere task join: {err}"))?;
Ok(if !holds {
DeleteOutcome::Incomplete("scope_changed")
} else if !zone_listed {
DeleteOutcome::Incomplete("zone_not_listed")
} else {
DeleteOutcome::Synced(count)
})
}
Some(report) => Ok(DeleteOutcome::Incomplete(outcome_tag(&report.outcome))),
None => Ok(DeleteOutcome::Deferred),
}
}
}